farcasclaudiu/Flowise
1
0
Fork 0
mirror of https://github.com/farcasclaudiu/Flowise.git synced 2026-06-30 11:00:27 +03:00
Code Issues Packages Projects Releases Wiki Activity

Chore/refractor (#4454)

Browse Source 
* markdown files and env examples cleanup

* components update

* update jsonlines description

* server refractor

* update telemetry

* add execute custom node

* add ui refractor

* add username and password authenticate

* correctly retrieve past images in agentflowv2

* disable e2e temporarily

* add existing username and password authenticate

* update migration to default workspace

* update todo

* blob storage migrating

* throw error on agent tool call error

* add missing execution import

* add referral

* chore: add error message when importData is undefined

* migrate api keys to db

* fix: data too long for column executionData

* migrate api keys from json to db at init

* add info on account setup

* update docstore missing fields

---------

Co-authored-by: chungyau97 <chungyau97@gmail.com>
This commit is contained in:
Henry Heng
2025-05-27 14:29:42 +08:00
committed by GitHub
parent e35a126b46
commit 5a37227d14
560 changed files with 62127 additions and 4100 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/components/nodes/agentflow/Agent/Agent.ts
+2 -1
View File
@@ -427,7 +427,8 @@ class Agent_Agentflow implements INode {
return returnData
}
const stores = await appDataSource.getRepository(databaseEntities['DocumentStore']).find()
const searchOptions = options.searchOptions || {}
const stores = await appDataSource.getRepository(databaseEntities['DocumentStore']).findBy(searchOptions)
for (const store of stores) {
if (store.status === 'UPSERTED') {
const obj = {
packages/components/nodes/agentflow/CustomFunction/CustomFunction.ts
+1 -1
View File
@@ -152,7 +152,7 @@ class CustomFunction_Agentflow implements INode {
newState = updateFlowState(state, _customFunctionUpdateState)
}
const variables = await getVars(appDataSource, databaseEntities, nodeData)
const variables = await getVars(appDataSource, databaseEntities, nodeData, options)
const flow = {
chatflowId: options.chatflowid,
sessionId: options.sessionId,
packages/components/nodes/agentflow/ExecuteFlow/ExecuteFlow.ts
+2 -1
View File
@@ -127,7 +127,8 @@ class ExecuteFlow_Agentflow implements INode {
return returnData
}
const chatflows = await appDataSource.getRepository(databaseEntities['ChatFlow']).find()
const searchOptions = options.searchOptions || {}
const chatflows = await appDataSource.getRepository(databaseEntities['ChatFlow']).findBy(searchOptions)
for (let i = 0; i < chatflows.length; i += 1) {
let cfType = 'Chatflow'
packages/components/nodes/agentflow/Retriever/Retriever.ts
+2 -1
View File
@@ -119,7 +119,8 @@ class Retriever_Agentflow implements INode {
return returnData
}
const stores = await appDataSource.getRepository(databaseEntities['DocumentStore']).find()
const searchOptions = options.searchOptions || {}
const stores = await appDataSource.getRepository(databaseEntities['DocumentStore']).findBy(searchOptions)
for (const store of stores) {
if (store.status === 'UPSERTED') {
const obj = {
packages/components/nodes/agentflow/utils.ts
+3 -3
View File
@@ -18,7 +18,7 @@ export const addImagesToMessages = async (
for (const upload of imageUploads) {
let bf = upload.data
if (upload.type == 'stored-file') {
const contents = await getFileFromStorage(upload.name, options.chatflowid, options.chatId)
const contents = await getFileFromStorage(upload.name, options.orgId, options.chatflowid, options.chatId)
// as the image is stored in the server, read the file and convert it to base64
bf = 'data:' + upload.mime + ';base64,' + contents.toString('base64')
@@ -90,7 +90,7 @@ export const processMessagesWithImages = async (
hasImageReferences = true
try {
// Get file contents from storage
const contents = await getFileFromStorage(item.name, options.chatflowid, options.chatId)
const contents = await getFileFromStorage(item.name, options.orgId, options.chatflowid, options.chatId)
// Create base64 data URL
const base64Data = 'data:' + item.mime + ';base64,' + contents.toString('base64')
@@ -319,7 +319,7 @@ export const getPastChatHistoryImageMessages = async (
const imageContents: MessageContentImageUrl[] = []
for (const upload of uploads) {
if (upload.type === 'stored-file' && upload.mime.startsWith('image/')) {
const fileData = await getFileFromStorage(upload.name, options.chatflowid, options.chatId)
const fileData = await getFileFromStorage(upload.name, options.orgId, options.chatflowid, options.chatId)
// as the image is stored in the server, read the file and convert it to base64
const bf = 'data:' + upload.mime + ';base64,' + fileData.toString('base64')
packages/components/nodes/agents/AirtableAgent/AirtableAgent.ts
+3 -3
View File
@@ -128,7 +128,7 @@ class Airtable_Agents implements INode {
let base64String = Buffer.from(JSON.stringify(airtableData)).toString('base64')
const loggerHandler = new ConsoleCallbackHandler(options.logger)
const loggerHandler = new ConsoleCallbackHandler(options.logger, options?.orgId)
const callbacks = await additionalCallbacks(nodeData, options)
const pyodide = await LoadPyodide()
@@ -163,7 +163,7 @@ json.dumps(my_dict)`
const chain = new LLMChain({
llm: model,
prompt: PromptTemplate.fromTemplate(systemPrompt),
verbose: process.env.DEBUG === 'true'
verbose: process.env.DEBUG === 'true' ? true : false
})
const inputs = {
dict: dataframeColDict,
@@ -192,7 +192,7 @@ json.dumps(my_dict)`
const chain = new LLMChain({
llm: model,
prompt: PromptTemplate.fromTemplate(finalSystemPrompt),
verbose: process.env.DEBUG === 'true'
verbose: process.env.DEBUG === 'true' ? true : false
})
const inputs = {
question: input,
packages/components/nodes/agents/CSVAgent/CSVAgent.ts
+5 -4
View File
@@ -97,7 +97,7 @@ class CSV_Agents implements INode {
}
}
const loggerHandler = new ConsoleCallbackHandler(options.logger)
const loggerHandler = new ConsoleCallbackHandler(options.logger, options?.orgId)
const shouldStreamResponse = options.shouldStreamResponse
const sseStreamer: IServerSideEventStreamer = options.sseStreamer as IServerSideEventStreamer
const chatId = options.chatId
@@ -114,11 +114,12 @@ class CSV_Agents implements INode {
} else {
files = [fileName]
}
const orgId = options.orgId
const chatflowid = options.chatflowid
for (const file of files) {
if (!file) continue
const fileData = await getFileFromStorage(file, chatflowid)
const fileData = await getFileFromStorage(file, orgId, chatflowid)
base64String += fileData.toString('base64')
}
} else {
@@ -170,7 +171,7 @@ json.dumps(my_dict)`
const chain = new LLMChain({
llm: model,
prompt: PromptTemplate.fromTemplate(systemPrompt),
verbose: process.env.DEBUG === 'true'
verbose: process.env.DEBUG === 'true' ? true : false
})
const inputs = {
dict: dataframeColDict,
@@ -201,7 +202,7 @@ json.dumps(my_dict)`
prompt: PromptTemplate.fromTemplate(
systemMessagePrompt ? `${systemMessagePrompt}\n${finalSystemPrompt}` : finalSystemPrompt
),
verbose: process.env.DEBUG === 'true'
verbose: process.env.DEBUG === 'true' ? true : false
})
const inputs = {
question: input,
packages/components/nodes/agents/ConversationalAgent/ConversationalAgent.ts
+1 -1
View File
@@ -132,7 +132,7 @@ class ConversationalAgent_Agents implements INode {
}
const executor = await prepareAgent(nodeData, options, { sessionId: this.sessionId, chatId: options.chatId, input })
const loggerHandler = new ConsoleCallbackHandler(options.logger)
const loggerHandler = new ConsoleCallbackHandler(options.logger, options?.orgId)
const callbacks = await additionalCallbacks(nodeData, options)
let res: ChainValues = {}
packages/components/nodes/agents/ConversationalRetrievalToolAgent/ConversationalRetrievalToolAgent.ts
+2 -2
View File
@@ -130,7 +130,7 @@ class ConversationalRetrievalToolAgent_Agents implements INode {
const executor = await prepareAgent(nodeData, options, { sessionId: this.sessionId, chatId: options.chatId, input })
const loggerHandler = new ConsoleCallbackHandler(options.logger)
const loggerHandler = new ConsoleCallbackHandler(options.logger, options?.orgId)
const callbacks = await additionalCallbacks(nodeData, options)
let res: ChainValues = {}
@@ -288,7 +288,7 @@ const prepareAgent = async (
sessionId: flowObj?.sessionId,
chatId: flowObj?.chatId,
input: flowObj?.input,
verbose: process.env.DEBUG === 'true',
verbose: process.env.DEBUG === 'true' ? true : false,
maxIterations: maxIterations ? parseFloat(maxIterations) : undefined
})
packages/components/nodes/agents/LlamaIndexAgents/AnthropicAgent/AnthropicAgent_LlamaIndex.ts
+6 -2
View File
@@ -2,6 +2,7 @@ import { flatten } from 'lodash'
import { MessageContentTextDetail, ChatMessage, AnthropicAgent, Anthropic } from 'llamaindex'
import { getBaseClasses } from '../../../../src/utils'
import { FlowiseMemory, ICommonObject, IMessage, INode, INodeData, INodeParams, IUsedTool } from '../../../../src/Interface'
import { EvaluationRunTracerLlama } from '../../../../evaluation/EvaluationRunTracerLlama'
class AnthropicAgent_LlamaIndex_Agents implements INode {
label: string
@@ -96,13 +97,16 @@ class AnthropicAgent_LlamaIndex_Agents implements INode {
tools,
llm: model,
chatHistory: chatHistory,
verbose: process.env.DEBUG === 'true'
verbose: process.env.DEBUG === 'true' ? true : false
})
// these are needed for evaluation runs
await EvaluationRunTracerLlama.injectEvaluationMetadata(nodeData, options, agent)
let text = ''
const usedTools: IUsedTool[] = []
const response = await agent.chat({ message: input, chatHistory, verbose: process.env.DEBUG === 'true' })
const response = await agent.chat({ message: input, chatHistory, verbose: process.env.DEBUG === 'true' ? true : false })
if (response.sources.length) {
for (const sourceTool of response.sources) {
packages/components/nodes/agents/LlamaIndexAgents/OpenAIToolAgent/OpenAIToolAgent_LlamaIndex.ts
+7 -4
View File
@@ -1,6 +1,7 @@
import { flatten } from 'lodash'
import { ChatMessage, OpenAI, OpenAIAgent } from 'llamaindex'
import { getBaseClasses } from '../../../../src/utils'
import { EvaluationRunTracerLlama } from '../../../../evaluation/EvaluationRunTracerLlama'
import {
FlowiseMemory,
ICommonObject,
@@ -107,9 +108,12 @@ class OpenAIFunctionAgent_LlamaIndex_Agents implements INode {
tools,
llm: model,
chatHistory: chatHistory,
verbose: process.env.DEBUG === 'true'
verbose: process.env.DEBUG === 'true' ? true : false
})
// these are needed for evaluation runs
await EvaluationRunTracerLlama.injectEvaluationMetadata(nodeData, options, agent)
let text = ''
let isStreamingStarted = false
const usedTools: IUsedTool[] = []
@@ -119,10 +123,9 @@ class OpenAIFunctionAgent_LlamaIndex_Agents implements INode {
message: input,
chatHistory,
stream: true,
verbose: process.env.DEBUG === 'true'
verbose: process.env.DEBUG === 'true' ? true : false
})
for await (const chunk of stream) {
//console.log('chunk', chunk)
text += chunk.response.delta
if (!isStreamingStarted) {
isStreamingStarted = true
@@ -147,7 +150,7 @@ class OpenAIFunctionAgent_LlamaIndex_Agents implements INode {
}
}
} else {
const response = await agent.chat({ message: input, chatHistory, verbose: process.env.DEBUG === 'true' })
const response = await agent.chat({ message: input, chatHistory, verbose: process.env.DEBUG === 'true' ? true : false })
if (response.sources.length) {
for (const sourceTool of response.sources) {
usedTools.push({
packages/components/nodes/agents/OpenAIAssistant/OpenAIAssistant.ts
+123 -20
View File
@@ -107,7 +107,11 @@ class OpenAIAssistant_Agents implements INode {
return returnData
}
const assistants = await appDataSource.getRepository(databaseEntities['Assistant']).find()
const searchOptions = options.searchOptions || {}
const assistants = await appDataSource.getRepository(databaseEntities['Assistant']).findBy({
...searchOptions,
type: 'OPENAI'
})
for (let i = 0; i < assistants.length; i += 1) {
const assistantDetails = JSON.parse(assistants[i].details)
@@ -130,13 +134,14 @@ class OpenAIAssistant_Agents implements INode {
const selectedAssistantId = nodeData.inputs?.selectedAssistant as string
const appDataSource = options.appDataSource as DataSource
const databaseEntities = options.databaseEntities as IDatabaseEntity
const orgId = options.orgId
const assistant = await appDataSource.getRepository(databaseEntities['Assistant']).findOneBy({
id: selectedAssistantId
})
if (!assistant) {
options.logger.error(`Assistant ${selectedAssistantId} not found`)
options.logger.error(`[${orgId}]: Assistant ${selectedAssistantId} not found`)
return
}
@@ -149,7 +154,7 @@ class OpenAIAssistant_Agents implements INode {
chatId
})
if (!chatmsg) {
options.logger.error(`Chat Message with Chat Id: ${chatId} not found`)
options.logger.error(`[${orgId}]: Chat Message with Chat Id: ${chatId} not found`)
return
}
sessionId = chatmsg.sessionId
@@ -160,21 +165,21 @@ class OpenAIAssistant_Agents implements INode {
const credentialData = await getCredentialData(assistant.credential ?? '', options)
const openAIApiKey = getCredentialParam('openAIApiKey', credentialData, nodeData)
if (!openAIApiKey) {
options.logger.error(`OpenAI ApiKey not found`)
options.logger.error(`[${orgId}]: OpenAI ApiKey not found`)
return
}
const openai = new OpenAI({ apiKey: openAIApiKey })
options.logger.info(`Clearing OpenAI Thread ${sessionId}`)
options.logger.info(`[${orgId}]: Clearing OpenAI Thread ${sessionId}`)
try {
if (sessionId && sessionId.startsWith('thread_')) {
await openai.beta.threads.del(sessionId)
options.logger.info(`Successfully cleared OpenAI Thread ${sessionId}`)
options.logger.info(`[${orgId}]: Successfully cleared OpenAI Thread ${sessionId}`)
} else {
options.logger.error(`Error clearing OpenAI Thread ${sessionId}`)
options.logger.error(`[${orgId}]: Error clearing OpenAI Thread ${sessionId}`)
}
} catch (e) {
options.logger.error(`Error clearing OpenAI Thread ${sessionId}`)
options.logger.error(`[${orgId}]: Error clearing OpenAI Thread ${sessionId}`)
}
}
@@ -190,6 +195,17 @@ class OpenAIAssistant_Agents implements INode {
const shouldStreamResponse = options.shouldStreamResponse
const sseStreamer: IServerSideEventStreamer = options.sseStreamer as IServerSideEventStreamer
const chatId = options.chatId
const checkStorage = options.checkStorage
? (options.checkStorage as (orgId: string, subscriptionId: string, usageCacheManager: any) => Promise<void>)
: undefined
const updateStorageUsage = options.updateStorageUsage
? (options.updateStorageUsage as (
orgId: string,
workspaceId: string,
totalSize: number,
usageCacheManager: any
) => Promise<void>)
: undefined
if (moderations && moderations.length > 0) {
try {
@@ -380,17 +396,30 @@ class OpenAIAssistant_Agents implements INode {
// eslint-disable-next-line no-useless-escape
const fileName = cited_file.filename.split(/[\/\\]/).pop() ?? cited_file.filename
if (!disableFileDownload) {
filePath = await downloadFile(
if (checkStorage)
await checkStorage(options.orgId, options.subscriptionId, options.usageCacheManager)
const { path, totalSize } = await downloadFile(
openAIApiKey,
cited_file,
fileName,
options.orgId,
options.chatflowid,
options.chatId
)
filePath = path
fileAnnotations.push({
filePath,
fileName
})
if (updateStorageUsage)
await updateStorageUsage(
options.orgId,
options.workspaceId,
totalSize,
options.usageCacheManager
)
}
} else {
const file_path = (annotation as OpenAI.Beta.Threads.Messages.FilePathAnnotation).file_path
@@ -399,17 +428,30 @@ class OpenAIAssistant_Agents implements INode {
// eslint-disable-next-line no-useless-escape
const fileName = cited_file.filename.split(/[\/\\]/).pop() ?? cited_file.filename
if (!disableFileDownload) {
filePath = await downloadFile(
if (checkStorage)
await checkStorage(options.orgId, options.subscriptionId, options.usageCacheManager)
const { path, totalSize } = await downloadFile(
openAIApiKey,
cited_file,
fileName,
options.orgId,
options.chatflowid,
options.chatId
)
filePath = path
fileAnnotations.push({
filePath,
fileName
})
if (updateStorageUsage)
await updateStorageUsage(
options.orgId,
options.workspaceId,
totalSize,
options.usageCacheManager
)
}
}
}
@@ -467,15 +509,21 @@ class OpenAIAssistant_Agents implements INode {
const fileId = chunk.image_file.file_id
const fileObj = await openai.files.retrieve(fileId)
const filePath = await downloadImg(
if (checkStorage) await checkStorage(options.orgId, options.subscriptionId, options.usageCacheManager)
const { filePath, totalSize } = await downloadImg(
openai,
fileId,
`${fileObj.filename}.png`,
options.orgId,
options.chatflowid,
options.chatId
)
artifacts.push({ type: 'png', data: filePath })
if (updateStorageUsage)
await updateStorageUsage(options.orgId, options.workspaceId, totalSize, options.usageCacheManager)
if (!isStreamingStarted) {
isStreamingStarted = true
if (sseStreamer) {
@@ -776,7 +824,21 @@ class OpenAIAssistant_Agents implements INode {
// eslint-disable-next-line no-useless-escape
const fileName = cited_file.filename.split(/[\/\\]/).pop() ?? cited_file.filename
if (!disableFileDownload) {
filePath = await downloadFile(openAIApiKey, cited_file, fileName, options.chatflowid, options.chatId)
if (checkStorage) await checkStorage(options.orgId, options.subscriptionId, options.usageCacheManager)
const { path, totalSize } = await downloadFile(
openAIApiKey,
cited_file,
fileName,
options.orgId,
options.chatflowid,
options.chatId
)
filePath = path
if (updateStorageUsage)
await updateStorageUsage(options.orgId, options.workspaceId, totalSize, options.usageCacheManager)
fileAnnotations.push({
filePath,
fileName
@@ -789,13 +851,27 @@ class OpenAIAssistant_Agents implements INode {
// eslint-disable-next-line no-useless-escape
const fileName = cited_file.filename.split(/[\/\\]/).pop() ?? cited_file.filename
if (!disableFileDownload) {
filePath = await downloadFile(
if (checkStorage)
await checkStorage(options.orgId, options.subscriptionId, options.usageCacheManager)
const { path, totalSize } = await downloadFile(
openAIApiKey,
cited_file,
fileName,
options.orgId,
options.chatflowid,
options.chatId
)
filePath = path
if (updateStorageUsage)
await updateStorageUsage(
options.orgId,
options.workspaceId,
totalSize,
options.usageCacheManager
)
fileAnnotations.push({
filePath,
fileName
@@ -822,7 +898,20 @@ class OpenAIAssistant_Agents implements INode {
const fileId = content.image_file.file_id
const fileObj = await openai.files.retrieve(fileId)
const filePath = await downloadImg(openai, fileId, `${fileObj.filename}.png`, options.chatflowid, options.chatId)
if (checkStorage) await checkStorage(options.orgId, options.subscriptionId, options.usageCacheManager)
const { filePath, totalSize } = await downloadImg(
openai,
fileId,
`${fileObj.filename}.png`,
options.orgId,
options.chatflowid,
options.chatId
)
if (updateStorageUsage)
await updateStorageUsage(options.orgId, options.workspaceId, totalSize, options.usageCacheManager)
artifacts.push({ type: 'png', data: filePath })
}
}
@@ -847,7 +936,13 @@ class OpenAIAssistant_Agents implements INode {
}
}
const downloadImg = async (openai: OpenAI, fileId: string, fileName: string, ...paths: string[]) => {
const downloadImg = async (
openai: OpenAI,
fileId: string,
fileName: string,
orgId: string,
...paths: string[]
): Promise<{ filePath: string; totalSize: number }> => {
const response = await openai.files.content(fileId)
// Extract the binary data from the Response object
@@ -857,12 +952,18 @@ const downloadImg = async (openai: OpenAI, fileId: string, fileName: string, ...
const image_data_buffer = Buffer.from(image_data)
const mime = 'image/png'
const res = await addSingleFileToStorage(mime, image_data_buffer, fileName, ...paths)
const { path, totalSize } = await addSingleFileToStorage(mime, image_data_buffer, fileName, orgId, ...paths)
return res
return { filePath: path, totalSize }
}
const downloadFile = async (openAIApiKey: string, fileObj: any, fileName: string, ...paths: string[]) => {
const downloadFile = async (
openAIApiKey: string,
fileObj: any,
fileName: string,
orgId: string,
...paths: string[]
): Promise<{ path: string; totalSize: number }> => {
try {
const response = await fetch(`https://api.openai.com/v1/files/${fileObj.id}/content`, {
method: 'GET',
@@ -880,10 +981,12 @@ const downloadFile = async (openAIApiKey: string, fileObj: any, fileName: string
const data_buffer = Buffer.from(data)
const mime = 'application/octet-stream'
return await addSingleFileToStorage(mime, data_buffer, fileName, ...paths)
const { path, totalSize } = await addSingleFileToStorage(mime, data_buffer, fileName, orgId, ...paths)
return { path, totalSize }
} catch (error) {
console.error('Error downloading or writing the file:', error)
return ''
return { path: '', totalSize: 0 }
}
}
packages/components/nodes/agents/ReActAgentLLM/ReActAgentLLM.ts
+1 -1
View File
@@ -97,7 +97,7 @@ class ReActAgentLLM_Agents implements INode {
const executor = new AgentExecutor({
agent,
tools,
verbose: process.env.DEBUG === 'true',
verbose: process.env.DEBUG === 'true' ? true : false,
maxIterations: maxIterations ? parseFloat(maxIterations) : undefined
})
packages/components/nodes/agents/ToolAgent/ToolAgent.ts
+2 -2
View File
@@ -143,7 +143,7 @@ class ToolAgent_Agents implements INode {
const executor = await prepareAgent(nodeData, options, { sessionId: this.sessionId, chatId: options.chatId, input })
const loggerHandler = new ConsoleCallbackHandler(options.logger)
const loggerHandler = new ConsoleCallbackHandler(options.logger, options?.orgId)
const callbacks = await additionalCallbacks(nodeData, options)
// Add custom streaming handler if detailed streaming is enabled
@@ -370,7 +370,7 @@ const prepareAgent = async (
sessionId: flowObj?.sessionId,
chatId: flowObj?.chatId,
input: flowObj?.input,
verbose: process.env.DEBUG === 'true',
verbose: process.env.DEBUG === 'true' ? true : false,
maxIterations: maxIterations ? parseFloat(maxIterations) : undefined
})
packages/components/nodes/agents/XMLAgent/XMLAgent.ts
+2 -2
View File
@@ -138,7 +138,7 @@ class XMLAgent_Agents implements INode {
}
const executor = await prepareAgent(nodeData, options, { sessionId: this.sessionId, chatId: options.chatId, input })
const loggerHandler = new ConsoleCallbackHandler(options.logger)
const loggerHandler = new ConsoleCallbackHandler(options.logger, options?.orgId)
const callbacks = await additionalCallbacks(nodeData, options)
let res: ChainValues = {}
@@ -278,7 +278,7 @@ const prepareAgent = async (
chatId: flowObj?.chatId,
input: flowObj?.input,
isXML: true,
verbose: process.env.DEBUG === 'true',
verbose: process.env.DEBUG === 'true' ? true : false,
maxIterations: maxIterations ? parseFloat(maxIterations) : undefined
})
packages/components/nodes/chains/ApiChain/GETApiChain.ts
+2 -2
View File
@@ -98,7 +98,7 @@ class GETApiChain_Chains implements INode {
const ansPrompt = nodeData.inputs?.ansPrompt as string
const chain = await getAPIChain(apiDocs, model, headers, urlPrompt, ansPrompt)
const loggerHandler = new ConsoleCallbackHandler(options.logger)
const loggerHandler = new ConsoleCallbackHandler(options.logger, options?.orgId)
const callbacks = await additionalCallbacks(nodeData, options)
const shouldStreamResponse = options.shouldStreamResponse
const sseStreamer: IServerSideEventStreamer = options.sseStreamer as IServerSideEventStreamer
@@ -129,7 +129,7 @@ const getAPIChain = async (documents: string, llm: BaseLanguageModel, headers: s
const chain = APIChain.fromLLMAndAPIDocs(llm, documents, {
apiUrlPrompt,
apiResponsePrompt,
verbose: process.env.DEBUG === 'true',
verbose: process.env.DEBUG === 'true' ? true : false,
headers: typeof headers === 'object' ? headers : headers ? JSON.parse(headers) : {}
})
return chain
packages/components/nodes/chains/ApiChain/OpenAPIChain.ts
+4 -3
View File
@@ -71,7 +71,7 @@ class OpenApiChain_Chains implements INode {
async run(nodeData: INodeData, input: string, options: ICommonObject): Promise<string | object> {
const chain = await initChain(nodeData, options)
const loggerHandler = new ConsoleCallbackHandler(options.logger)
const loggerHandler = new ConsoleCallbackHandler(options.logger, options?.orgId)
const callbacks = await additionalCallbacks(nodeData, options)
const moderations = nodeData.inputs?.inputModeration as Moderation[]
const shouldStreamResponse = options.shouldStreamResponse
@@ -114,8 +114,9 @@ const initChain = async (nodeData: INodeData, options: ICommonObject) => {
} else {
if (yamlFileBase64.startsWith('FILE-STORAGE::')) {
const file = yamlFileBase64.replace('FILE-STORAGE::', '')
const orgId = options.orgId
const chatflowid = options.chatflowid
const fileData = await getFileFromStorage(file, chatflowid)
const fileData = await getFileFromStorage(file, orgId, chatflowid)
yamlString = fileData.toString()
} else {
const splitDataURI = yamlFileBase64.split(',')
@@ -128,7 +129,7 @@ const initChain = async (nodeData: INodeData, options: ICommonObject) => {
return await createOpenAPIChain(yamlString, {
llm: model,
headers: typeof headers === 'object' ? headers : headers ? JSON.parse(headers) : {},
verbose: process.env.DEBUG === 'true'
verbose: process.env.DEBUG === 'true' ? true : false
})
}
packages/components/nodes/chains/ApiChain/POSTApiChain.ts
+2 -2
View File
@@ -87,7 +87,7 @@ class POSTApiChain_Chains implements INode {
const ansPrompt = nodeData.inputs?.ansPrompt as string
const chain = await getAPIChain(apiDocs, model, headers, urlPrompt, ansPrompt)
const loggerHandler = new ConsoleCallbackHandler(options.logger)
const loggerHandler = new ConsoleCallbackHandler(options.logger, options?.orgId)
const callbacks = await additionalCallbacks(nodeData, options)
const shouldStreamResponse = options.shouldStreamResponse
@@ -119,7 +119,7 @@ const getAPIChain = async (documents: string, llm: BaseLanguageModel, headers: s
const chain = APIChain.fromLLMAndAPIDocs(llm, documents, {
apiUrlPrompt,
apiResponsePrompt,
verbose: process.env.DEBUG === 'true',
verbose: process.env.DEBUG === 'true' ? true : false,
headers: typeof headers === 'object' ? headers : headers ? JSON.parse(headers) : {}
})
return chain
packages/components/nodes/chains/ConversationChain/ConversationChain.ts
+1 -1
View File
@@ -132,7 +132,7 @@ class ConversationChain_Chains implements INode {
}
}
const loggerHandler = new ConsoleCallbackHandler(options.logger)
const loggerHandler = new ConsoleCallbackHandler(options.logger, options?.orgId)
const additionalCallback = await additionalCallbacks(nodeData, options)
let res = ''
packages/components/nodes/chains/ConversationalRetrievalQAChain/ConversationalRetrievalQAChain.ts
+8 -3
View File
@@ -185,6 +185,7 @@ class ConversationalRetrievalQAChain_Chains implements INode {
const shouldStreamResponse = options.shouldStreamResponse
const sseStreamer: IServerSideEventStreamer = options.sseStreamer as IServerSideEventStreamer
const chatId = options.chatId
const orgId = options.orgId
let customResponsePrompt = responsePrompt
// If the deprecated systemMessagePrompt is still exists
@@ -200,7 +201,8 @@ class ConversationalRetrievalQAChain_Chains implements INode {
memoryKey: 'chat_history',
appDataSource,
databaseEntities,
chatflowid
chatflowid,
orgId
})
}
@@ -220,7 +222,7 @@ class ConversationalRetrievalQAChain_Chains implements INode {
const history = ((await memory.getChatMessages(this.sessionId, false, prependMessages)) as IMessage[]) ?? []
const loggerHandler = new ConsoleCallbackHandler(options.logger)
const loggerHandler = new ConsoleCallbackHandler(options.logger, options?.orgId)
const additionalCallback = await additionalCallbacks(nodeData, options)
let callbacks = [loggerHandler, ...additionalCallback]
@@ -407,18 +409,21 @@ interface BufferMemoryExtendedInput {
appDataSource: DataSource
databaseEntities: IDatabaseEntity
chatflowid: string
orgId: string
}
class BufferMemory extends FlowiseMemory implements MemoryMethods {
appDataSource: DataSource
databaseEntities: IDatabaseEntity
chatflowid: string
orgId: string
constructor(fields: BufferMemoryInput & BufferMemoryExtendedInput) {
super(fields)
this.appDataSource = fields.appDataSource
this.databaseEntities = fields.databaseEntities
this.chatflowid = fields.chatflowid
this.orgId = fields.orgId
}
async getChatMessages(
@@ -443,7 +448,7 @@ class BufferMemory extends FlowiseMemory implements MemoryMethods {
}
if (returnBaseMessages) {
return await mapChatMessageToBaseMessage(chatMessage)
return await mapChatMessageToBaseMessage(chatMessage, this.orgId)
}
let returnIMessages: IMessage[] = []
packages/components/nodes/chains/GraphCypherQAChain/GraphCypherQAChain.ts
+1 -1
View File
@@ -215,7 +215,7 @@ class GraphCypherQA_Chain implements INode {
query: input
}
const loggerHandler = new ConsoleCallbackHandler(options.logger)
const loggerHandler = new ConsoleCallbackHandler(options.logger, options?.orgId)
const callbackHandlers = await additionalCallbacks(nodeData, options)
let callbacks = [loggerHandler, ...callbackHandlers]
packages/components/nodes/chains/LLMChain/LLMChain.ts
+1 -1
View File
@@ -167,7 +167,7 @@ const runPrediction = async (
nodeData: INodeData,
disableStreaming?: boolean
) => {
const loggerHandler = new ConsoleCallbackHandler(options.logger)
const loggerHandler = new ConsoleCallbackHandler(options.logger, options?.orgId)
const callbacks = await additionalCallbacks(nodeData, options)
const moderations = nodeData.inputs?.inputModeration as Moderation[]
packages/components/nodes/chains/MultiPromptChain/MultiPromptChain.ts
+2 -2
View File
@@ -66,7 +66,7 @@ class MultiPromptChain_Chains implements INode {
promptNames,
promptDescriptions,
promptTemplates,
llmChainOpts: { verbose: process.env.DEBUG === 'true' }
llmChainOpts: { verbose: process.env.DEBUG === 'true' ? true : false }
})
return chain
@@ -95,7 +95,7 @@ class MultiPromptChain_Chains implements INode {
}
const obj = { input }
const loggerHandler = new ConsoleCallbackHandler(options.logger)
const loggerHandler = new ConsoleCallbackHandler(options.logger, options?.orgId)
const callbacks = await additionalCallbacks(nodeData, options)
if (shouldStreamResponse) {
packages/components/nodes/chains/MultiRetrievalQAChain/MultiRetrievalQAChain.ts
+2 -2
View File
@@ -74,7 +74,7 @@ class MultiRetrievalQAChain_Chains implements INode {
retrieverNames,
retrieverDescriptions,
retrievers,
retrievalQAChainOpts: { verbose: process.env.DEBUG === 'true', returnSourceDocuments }
retrievalQAChainOpts: { verbose: process.env.DEBUG === 'true' ? true : false, returnSourceDocuments }
})
return chain
}
@@ -101,7 +101,7 @@ class MultiRetrievalQAChain_Chains implements INode {
}
}
const obj = { input }
const loggerHandler = new ConsoleCallbackHandler(options.logger)
const loggerHandler = new ConsoleCallbackHandler(options.logger, options?.orgId)
const callbacks = await additionalCallbacks(nodeData, options)
if (shouldStreamResponse) {
packages/components/nodes/chains/RetrievalQAChain/RetrievalQAChain.ts
+2 -2
View File
@@ -53,7 +53,7 @@ class RetrievalQAChain_Chains implements INode {
const model = nodeData.inputs?.model as BaseLanguageModel
const vectorStoreRetriever = nodeData.inputs?.vectorStoreRetriever as BaseRetriever
const chain = RetrievalQAChain.fromLLM(model, vectorStoreRetriever, { verbose: process.env.DEBUG === 'true' })
const chain = RetrievalQAChain.fromLLM(model, vectorStoreRetriever, { verbose: process.env.DEBUG === 'true' ? true : false })
return chain
}
@@ -80,7 +80,7 @@ class RetrievalQAChain_Chains implements INode {
const obj = {
query: input
}
const loggerHandler = new ConsoleCallbackHandler(options.logger)
const loggerHandler = new ConsoleCallbackHandler(options.logger, options?.orgId)
const callbacks = await additionalCallbacks(nodeData, options)
if (shouldStreamResponse) {
packages/components/nodes/chains/SqlDatabaseChain/SqlDatabaseChain.ts
+2 -2
View File
@@ -194,7 +194,7 @@ class SqlDatabaseChain_Chains implements INode {
topK,
customPrompt
)
const loggerHandler = new ConsoleCallbackHandler(options.logger)
const loggerHandler = new ConsoleCallbackHandler(options.logger, options?.orgId)
const callbacks = await additionalCallbacks(nodeData, options)
if (shouldStreamResponse) {
@@ -241,7 +241,7 @@ const getSQLDBChain = async (
const obj: SqlDatabaseChainInput = {
llm,
database: db,
verbose: process.env.DEBUG === 'true',
verbose: process.env.DEBUG === 'true' ? true : false,
topK: topK
}
packages/components/nodes/chains/VectorDBQAChain/VectorDBQAChain.ts
+2 -2
View File
@@ -55,7 +55,7 @@ class VectorDBQAChain_Chains implements INode {
const chain = VectorDBQAChain.fromLLM(model, vectorStore, {
k: (vectorStore as any)?.k ?? 4,
verbose: process.env.DEBUG === 'true'
verbose: process.env.DEBUG === 'true' ? true : false
})
return chain
}
@@ -84,7 +84,7 @@ class VectorDBQAChain_Chains implements INode {
query: input
}
const loggerHandler = new ConsoleCallbackHandler(options.logger)
const loggerHandler = new ConsoleCallbackHandler(options.logger, options?.orgId)
const callbacks = await additionalCallbacks(nodeData, options)
if (shouldStreamResponse) {
packages/components/nodes/chatmodels/AzureChatOpenAI/README.md
+7 -7
View File
@@ -4,13 +4,13 @@ Azure OpenAI Chat Model integration for Flowise
## 🌱 Env Variables
| Variable | Description | Type | Default |
| ---------------------------- | ----------------------------------------------------------------------------------------------- | ------------------------------------------------ | ----------------------------------- |
| AZURE_OPENAI_API_KEY | Default `credential.azureOpenAIApiKey` for Azure OpenAI Model | String | |
| AZURE_OPENAI_API_INSTANCE_NAME | Default `credential.azureOpenAIApiInstanceName` for Azure OpenAI Model | String | |
| AZURE_OPENAI_API_DEPLOYMENT_NAME | Default `credential.azureOpenAIApiDeploymentName` for Azure OpenAI Model | String | |
| AZURE_OPENAI_API_VERSION | Default `credential.azureOpenAIApiVersion` for Azure OpenAI Model | String | |
| Variable | Description | Type | Default |
| -------------------------------- | ------------------------------------------------------------------------ | ------ | ------- |
| AZURE_OPENAI_API_KEY | Default `credential.azureOpenAIApiKey` for Azure OpenAI Model | String | |
| AZURE_OPENAI_API_INSTANCE_NAME | Default `credential.azureOpenAIApiInstanceName` for Azure OpenAI Model | String | |
| AZURE_OPENAI_API_DEPLOYMENT_NAME | Default `credential.azureOpenAIApiDeploymentName` for Azure OpenAI Model | String | |
| AZURE_OPENAI_API_VERSION | Default `credential.azureOpenAIApiVersion` for Azure OpenAI Model | String | |
## License
Source code in this repository is made available under the [Apache License Version 2.0](https://github.com/FlowiseAI/Flowise/blob/master/LICENSE.md).
Source code in this repository is made available under the [Apache License Version 2.0](https://github.com/FlowiseAI/Flowise/blob/master/LICENSE.md).
packages/components/nodes/chatmodels/ChatGoogleGenerativeAI/ChatGoogleGenerativeAI.ts
+4
View File
@@ -216,6 +216,10 @@ class GoogleGenerativeAI_ChatModels implements INode {
streaming: streaming ?? true
}
// this extra metadata is needed, as langchain does not show the model name in the callbacks.
obj.metadata = {
fw_model_name: customModelName || modelName
}
if (maxOutputTokens) obj.maxOutputTokens = parseInt(maxOutputTokens, 10)
if (topP) obj.topP = parseFloat(topP)
if (topK) obj.topK = parseFloat(topK)
packages/components/nodes/chatmodels/ChatIBMWatsonx/ChatIBMWatsonx.ts
+3 -2
View File
@@ -161,12 +161,13 @@ class ChatIBMWatsonx_ChatModels implements INode {
watsonxAIBearerToken
}
const obj: ChatWatsonxInput & WatsonxAuth = {
const obj = {
...auth,
streaming: streaming ?? true,
model: modelName,
temperature: temperature ? parseFloat(temperature) : undefined
}
} as ChatWatsonxInput & WatsonxAuth
if (cache) obj.cache = cache
if (maxTokens) obj.maxTokens = parseInt(maxTokens, 10)
if (frequencyPenalty) obj.frequencyPenalty = parseInt(frequencyPenalty, 10)
packages/components/nodes/documentloaders/Cheerio/Cheerio.ts
+12 -6
View File
@@ -123,6 +123,7 @@ class Cheerio_DocumentLoaders implements INode {
const selectedLinks = nodeData.inputs?.selectedLinks as string[]
let limit = parseInt(nodeData.inputs?.limit as string)
const output = nodeData.outputs?.output as string
const orgId = options.orgId
const _omitMetadataKeys = nodeData.inputs?.omitMetadataKeys as string
@@ -149,7 +150,8 @@ class Cheerio_DocumentLoaders implements INode {
try {
let docs: IDocument[] = []
if (url.endsWith('.pdf')) {
if (process.env.DEBUG === 'true') options.logger.info(`CheerioWebBaseLoader does not support PDF files: ${url}`)
if (process.env.DEBUG === 'true')
options.logger.info(`[${orgId}]: CheerioWebBaseLoader does not support PDF files: ${url}`)
return docs
}
const loader = new CheerioWebBaseLoader(url, params)
@@ -161,7 +163,8 @@ class Cheerio_DocumentLoaders implements INode {
}
return docs
} catch (err) {
if (process.env.DEBUG === 'true') options.logger.error(`error in CheerioWebBaseLoader: ${err.message}, on page: ${url}`)
if (process.env.DEBUG === 'true')
options.logger.error(`[${orgId}]: Error in CheerioWebBaseLoader: ${err.message}, on page: ${url}`)
return []
}
}
@@ -169,7 +172,7 @@ class Cheerio_DocumentLoaders implements INode {
let docs: IDocument[] = []
if (relativeLinksMethod) {
if (process.env.DEBUG === 'true') options.logger.info(`Start ${relativeLinksMethod}`)
if (process.env.DEBUG === 'true') options.logger.info(`[${orgId}]: Start CheerioWebBaseLoader ${relativeLinksMethod}`)
// if limit is 0 we don't want it to default to 10 so we check explicitly for null or undefined
// so when limit is 0 we can fetch all the links
if (limit === null || limit === undefined) limit = 10
@@ -180,15 +183,18 @@ class Cheerio_DocumentLoaders implements INode {
: relativeLinksMethod === 'webCrawl'
? await webCrawl(url, limit)
: await xmlScrape(url, limit)
if (process.env.DEBUG === 'true') options.logger.info(`pages: ${JSON.stringify(pages)}, length: ${pages.length}`)
if (process.env.DEBUG === 'true')
options.logger.info(`[${orgId}]: CheerioWebBaseLoader pages: ${JSON.stringify(pages)}, length: ${pages.length}`)
if (!pages || pages.length === 0) throw new Error('No relative links found')
for (const page of pages) {
docs.push(...(await cheerioLoader(page)))
}
if (process.env.DEBUG === 'true') options.logger.info(`Finish ${relativeLinksMethod}`)
if (process.env.DEBUG === 'true') options.logger.info(`[${orgId}]: Finish CheerioWebBaseLoader ${relativeLinksMethod}`)
} else if (selectedLinks && selectedLinks.length > 0) {
if (process.env.DEBUG === 'true')
options.logger.info(`pages: ${JSON.stringify(selectedLinks)}, length: ${selectedLinks.length}`)
options.logger.info(
`[${orgId}]: CheerioWebBaseLoader pages: ${JSON.stringify(selectedLinks)}, length: ${selectedLinks.length}`
)
for (const page of selectedLinks.slice(0, limit)) {
docs.push(...(await cheerioLoader(page)))
}
packages/components/nodes/documentloaders/Csv/Csv.ts
+4 -3
View File
@@ -107,9 +107,9 @@ class Csv_DocumentLoaders implements INode {
return { files, fromStorage }
}
async getFileData(file: string, { chatflowid }: { chatflowid: string }, fromStorage?: boolean) {
async getFileData(file: string, { orgId, chatflowid }: { orgId: string; chatflowid: string }, fromStorage?: boolean) {
if (fromStorage) {
return getFileFromStorage(file, chatflowid)
return getFileFromStorage(file, orgId, chatflowid)
} else {
const splitDataURI = file.split(',')
splitDataURI.pop()
@@ -126,6 +126,7 @@ class Csv_DocumentLoaders implements INode {
let docs: IDocument[] = []
const orgId = options.orgId
const chatflowid = options.chatflowid
const { files, fromStorage } = this.getFiles(nodeData)
@@ -133,7 +134,7 @@ class Csv_DocumentLoaders implements INode {
for (const file of files) {
if (!file) continue
const fileData = await this.getFileData(file, { chatflowid }, fromStorage)
const fileData = await this.getFileData(file, { orgId, chatflowid }, fromStorage)
const blob = new Blob([fileData])
const loader = new CSVLoader(blob, columnName.trim().length === 0 ? undefined : columnName.trim())
packages/components/nodes/documentloaders/CustomDocumentLoader/CustomDocumentLoader.ts
+1 -1
View File
@@ -72,7 +72,7 @@ class CustomDocumentLoader_DocumentLoaders implements INode {
const appDataSource = options.appDataSource as DataSource
const databaseEntities = options.databaseEntities as IDatabaseEntity
const variables = await getVars(appDataSource, databaseEntities, nodeData)
const variables = await getVars(appDataSource, databaseEntities, nodeData, options)
const flow = {
chatflowId: options.chatflowid,
sessionId: options.sessionId,
packages/components/nodes/documentloaders/DocumentStore/DocStoreLoader.ts
+2 -1
View File
@@ -60,7 +60,8 @@ class DocStore_DocumentLoaders implements INode {
return returnData
}
const stores = await appDataSource.getRepository(databaseEntities['DocumentStore']).find()
const searchOptions = options.searchOptions || {}
const stores = await appDataSource.getRepository(databaseEntities['DocumentStore']).findBy(searchOptions)
for (const store of stores) {
if (store.status === 'SYNC') {
const obj = {
packages/components/nodes/documentloaders/Docx/Docx.ts
+2 -1
View File
@@ -96,11 +96,12 @@ class Docx_DocumentLoaders implements INode {
} else {
files = [fileName]
}
const orgId = options.orgId
const chatflowid = options.chatflowid
for (const file of files) {
if (!file) continue
const fileData = await getFileFromStorage(file, chatflowid)
const fileData = await getFileFromStorage(file, orgId, chatflowid)
const blob = new Blob([fileData])
const loader = new DocxLoader(blob)
packages/components/nodes/documentloaders/Epub/Epub.ts
+2 -1
View File
@@ -118,10 +118,11 @@ class Epub_DocumentLoaders implements INode {
files = fileName.startsWith('[') && fileName.endsWith(']') ? JSON.parse(fileName) : [fileName]
const chatflowid = options.chatflowid
const orgId = options.orgId
for (const file of files) {
if (!file) continue
const fileData = await getFileFromStorage(file, chatflowid)
const fileData = await getFileFromStorage(file, orgId, chatflowid)
const tempFilePath = path.join(tempDir, `${Date.now()}_${file}`)
fs.writeFileSync(tempFilePath, fileData)
await this.extractDocs(usage, tempFilePath, textSplitter, docs)
packages/components/nodes/documentloaders/File/File.ts
+3 -2
View File
@@ -144,6 +144,7 @@ class File_DocumentLoaders implements INode {
} else {
files = [fileName]
}
const orgId = options.orgId
const chatflowid = options.chatflowid
// specific to createAttachment to get files from chatId
@@ -151,14 +152,14 @@ class File_DocumentLoaders implements INode {
if (retrieveAttachmentChatId) {
for (const file of files) {
if (!file) continue
const fileData = await getFileFromStorage(file, chatflowid, options.chatId)
const fileData = await getFileFromStorage(file, orgId, chatflowid, options.chatId)
const blob = new Blob([fileData])
fileBlobs.push({ blob, ext: file.split('.').pop() || '' })
}
} else {
for (const file of files) {
if (!file) continue
const fileData = await getFileFromStorage(file, chatflowid)
const fileData = await getFileFromStorage(file, orgId, chatflowid)
const blob = new Blob([fileData])
fileBlobs.push({ blob, ext: file.split('.').pop() || '' })
}
packages/components/nodes/documentloaders/Json/Json.ts
+2 -1
View File
@@ -146,11 +146,12 @@ class Json_DocumentLoaders implements INode {
} else {
files = [fileName]
}
const orgId = options.orgId
const chatflowid = options.chatflowid
for (const file of files) {
if (!file) continue
const fileData = await getFileFromStorage(file, chatflowid)
const fileData = await getFileFromStorage(file, orgId, chatflowid)
const blob = new Blob([fileData])
const loader = new JSONLoader(blob, pointers.length != 0 ? pointers : undefined, metadata)
packages/components/nodes/documentloaders/Jsonlines/Jsonlines.ts
+2 -1
View File
@@ -135,11 +135,12 @@ class Jsonlines_DocumentLoaders implements INode {
} else {
files = [fileName]
}
const orgId = options.orgId
const chatflowid = options.chatflowid
for (const file of files) {
if (!file) continue
const fileData = await getFileFromStorage(file, chatflowid)
const fileData = await getFileFromStorage(file, orgId, chatflowid)
const blob = new Blob([fileData])
const loader = new JSONLinesLoader(blob, pointer, metadata)
packages/components/nodes/documentloaders/Pdf/Pdf.ts
+2 -1
View File
@@ -122,11 +122,12 @@ class Pdf_DocumentLoaders implements INode {
} else {
files = [fileName]
}
const orgId = options.orgId
const chatflowid = options.chatflowid
for (const file of files) {
if (!file) continue
const fileData = await getFileFromStorage(file, chatflowid)
const fileData = await getFileFromStorage(file, orgId, chatflowid)
const bf = Buffer.from(fileData)
await this.extractDocs(usage, bf, legacyBuild, textSplitter, docs)
}
packages/components/nodes/documentloaders/Playwright/Playwright.ts
+10 -5
View File
@@ -159,6 +159,7 @@ class Playwright_DocumentLoaders implements INode {
let waitForSelector = nodeData.inputs?.waitForSelector as string
const _omitMetadataKeys = nodeData.inputs?.omitMetadataKeys as string
const output = nodeData.outputs?.output as string
const orgId = options.orgId
let omitMetadataKeys: string[] = []
if (_omitMetadataKeys) {
@@ -202,13 +203,14 @@ class Playwright_DocumentLoaders implements INode {
}
return docs
} catch (err) {
if (process.env.DEBUG === 'true') options.logger.error(`error in PlaywrightWebBaseLoader: ${err.message}, on page: ${url}`)
if (process.env.DEBUG === 'true')
options.logger.error(`[${orgId}]: Error in PlaywrightWebBaseLoader: ${err.message}, on page: ${url}`)
}
}
let docs: IDocument[] = []
if (relativeLinksMethod) {
if (process.env.DEBUG === 'true') options.logger.info(`Start ${relativeLinksMethod}`)
if (process.env.DEBUG === 'true') options.logger.info(`[${orgId}]: Start PlaywrightWebBaseLoader ${relativeLinksMethod}`)
// if limit is 0 we don't want it to default to 10 so we check explicitly for null or undefined
// so when limit is 0 we can fetch all the links
if (limit === null || limit === undefined) limit = 10
@@ -219,15 +221,18 @@ class Playwright_DocumentLoaders implements INode {
: relativeLinksMethod === 'webCrawl'
? await webCrawl(url, limit)
: await xmlScrape(url, limit)
if (process.env.DEBUG === 'true') options.logger.info(`pages: ${JSON.stringify(pages)}, length: ${pages.length}`)
if (process.env.DEBUG === 'true')
options.logger.info(`[${orgId}]: PlaywrightWebBaseLoader pages: ${JSON.stringify(pages)}, length: ${pages.length}`)
if (!pages || pages.length === 0) throw new Error('No relative links found')
for (const page of pages) {
docs.push(...(await playwrightLoader(page)))
}
if (process.env.DEBUG === 'true') options.logger.info(`Finish ${relativeLinksMethod}`)
if (process.env.DEBUG === 'true') options.logger.info(`[${orgId}]: Finish PlaywrightWebBaseLoader ${relativeLinksMethod}`)
} else if (selectedLinks && selectedLinks.length > 0) {
if (process.env.DEBUG === 'true')
options.logger.info(`pages: ${JSON.stringify(selectedLinks)}, length: ${selectedLinks.length}`)
options.logger.info(
`[${orgId}]: PlaywrightWebBaseLoader pages: ${JSON.stringify(selectedLinks)}, length: ${selectedLinks.length}`
)
for (const page of selectedLinks.slice(0, limit)) {
docs.push(...(await playwrightLoader(page)))
}
packages/components/nodes/documentloaders/Puppeteer/Puppeteer.ts
+10 -5
View File
@@ -155,6 +155,7 @@ class Puppeteer_DocumentLoaders implements INode {
let waitForSelector = nodeData.inputs?.waitForSelector as string
const _omitMetadataKeys = nodeData.inputs?.omitMetadataKeys as string
const output = nodeData.outputs?.output as string
const orgId = options.orgId
let omitMetadataKeys: string[] = []
if (_omitMetadataKeys) {
@@ -198,13 +199,14 @@ class Puppeteer_DocumentLoaders implements INode {
}
return docs
} catch (err) {
if (process.env.DEBUG === 'true') options.logger.error(`error in PuppeteerWebBaseLoader: ${err.message}, on page: ${url}`)
if (process.env.DEBUG === 'true')
options.logger.error(`[${orgId}]: Error in PuppeteerWebBaseLoader: ${err.message}, on page: ${url}`)
}
}
let docs: IDocument[] = []
if (relativeLinksMethod) {
if (process.env.DEBUG === 'true') options.logger.info(`Start ${relativeLinksMethod}`)
if (process.env.DEBUG === 'true') options.logger.info(`[${orgId}]: Start PuppeteerWebBaseLoader ${relativeLinksMethod}`)
// if limit is 0 we don't want it to default to 10 so we check explicitly for null or undefined
// so when limit is 0 we can fetch all the links
if (limit === null || limit === undefined) limit = 10
@@ -215,15 +217,18 @@ class Puppeteer_DocumentLoaders implements INode {
: relativeLinksMethod === 'webCrawl'
? await webCrawl(url, limit)
: await xmlScrape(url, limit)
if (process.env.DEBUG === 'true') options.logger.info(`pages: ${JSON.stringify(pages)}, length: ${pages.length}`)
if (process.env.DEBUG === 'true')
options.logger.info(`[${orgId}]: PuppeteerWebBaseLoader pages: ${JSON.stringify(pages)}, length: ${pages.length}`)
if (!pages || pages.length === 0) throw new Error('No relative links found')
for (const page of pages) {
docs.push(...(await puppeteerLoader(page)))
}
if (process.env.DEBUG === 'true') options.logger.info(`Finish ${relativeLinksMethod}`)
if (process.env.DEBUG === 'true') options.logger.info(`[${orgId}]: Finish PuppeteerWebBaseLoader ${relativeLinksMethod}`)
} else if (selectedLinks && selectedLinks.length > 0) {
if (process.env.DEBUG === 'true')
options.logger.info(`pages: ${JSON.stringify(selectedLinks)}, length: ${selectedLinks.length}`)
options.logger.info(
`[${orgId}]: PuppeteerWebBaseLoader pages: ${JSON.stringify(selectedLinks)}, length: ${selectedLinks.length}`
)
for (const page of selectedLinks.slice(0, limit)) {
docs.push(...(await puppeteerLoader(page)))
}
packages/components/nodes/documentloaders/S3File/README.md
+4 -4
View File
@@ -4,10 +4,10 @@ DS File Loarder integration for Flowise
## 🌱 Env Variables
| Variable | Description | Type | Default |
| ---------------------------- | ----------------------------------------------------------------------------------------------- | ------------------------------------------------ | ----------------------------------- |
| UNSTRUCTURED_API_URL | Default `unstructuredApiUrl` for S3 File Loader | String | http://localhost:8000/general/v0/general |
| Variable | Description | Type | Default |
| -------------------- | ----------------------------------------------- | ------ | ---------------------------------------- |
| UNSTRUCTURED_API_URL | Default `unstructuredApiUrl` for S3 File Loader | String | http://localhost:8000/general/v0/general |
## License
Source code in this repository is made available under the [Apache License Version 2.0](https://github.com/FlowiseAI/Flowise/blob/master/LICENSE.md).
Source code in this repository is made available under the [Apache License Version 2.0](https://github.com/FlowiseAI/Flowise/blob/master/LICENSE.md).
packages/components/nodes/documentloaders/Text/Text.ts
+2 -1
View File
@@ -98,11 +98,12 @@ class Text_DocumentLoaders implements INode {
} else {
files = [fileName]
}
const orgId = options.orgId
const chatflowid = options.chatflowid
for (const file of files) {
if (!file) continue
const fileData = await getFileFromStorage(file, chatflowid)
const fileData = await getFileFromStorage(file, orgId, chatflowid)
const blob = new Blob([fileData])
const loader = new TextLoader(blob)
packages/components/nodes/documentloaders/Unstructured/README.md
+4 -4
View File
@@ -4,10 +4,10 @@ Unstructured File Loader integration for Flowise
## 🌱 Env Variables
| Variable | Description | Type | Default |
| ---------------------------- | ----------------------------------------------------------------------------------------------- | ------------------------------------------------ | ----------------------------------- |
| UNSTRUCTURED_API_URL | Default `apiUrl` for Unstructured File/Floder Loader | String | http://localhost:8000/general/v0/general |
| Variable | Description | Type | Default |
| -------------------- | ---------------------------------------------------- | ------ | ---------------------------------------- |
| UNSTRUCTURED_API_URL | Default `apiUrl` for Unstructured File/Floder Loader | String | http://localhost:8000/general/v0/general |
## License
Source code in this repository is made available under the [Apache License Version 2.0](https://github.com/FlowiseAI/Flowise/blob/master/LICENSE.md).
Source code in this repository is made available under the [Apache License Version 2.0](https://github.com/FlowiseAI/Flowise/blob/master/LICENSE.md).
packages/components/nodes/documentloaders/Unstructured/UnstructuredFile.ts
+2 -1
View File
@@ -532,11 +532,12 @@ class UnstructuredFile_DocumentLoaders implements INode {
} else {
files = [fileName]
}
const orgId = options.orgId
const chatflowid = options.chatflowid
for (const file of files) {
if (!file) continue
const fileData = await getFileFromStorage(file, chatflowid)
const fileData = await getFileFromStorage(file, orgId, chatflowid)
const loaderDocs = await loader.loadAndSplitBuffer(fileData, file)
docs.push(...loaderDocs)
}
packages/components/nodes/embeddings/AzureOpenAIEmbedding/README.md
+7 -7
View File
@@ -4,13 +4,13 @@ Azure OpenAI Embedding Model integration for Flowise
## 🌱 Env Variables
| Variable | Description | Type | Default |
| ---------------------------- | ----------------------------------------------------------------------------------------------- | ------------------------------------------------ | ----------------------------------- |
| AZURE_OPENAI_API_KEY | Default `credential.azureOpenAIApiKey` for Azure OpenAI Model | String | |
| AZURE_OPENAI_API_INSTANCE_NAME | Default `credential.azureOpenAIApiInstanceName` for Azure OpenAI Model | String | |
| AZURE_OPENAI_API_EMBEDDINGS_DEPLOYMENT_NAME | Default `credential.azureOpenAIApiDeploymentName` for Azure OpenAI Model | String | |
| AZURE_OPENAI_API_VERSION | Default `credential.azureOpenAIApiVersion` for Azure OpenAI Model | String | |
| Variable | Description | Type | Default |
| ------------------------------------------- | ------------------------------------------------------------------------ | ------ | ------- |
| AZURE_OPENAI_API_KEY | Default `credential.azureOpenAIApiKey` for Azure OpenAI Model | String | |
| AZURE_OPENAI_API_INSTANCE_NAME | Default `credential.azureOpenAIApiInstanceName` for Azure OpenAI Model | String | |
| AZURE_OPENAI_API_EMBEDDINGS_DEPLOYMENT_NAME | Default `credential.azureOpenAIApiDeploymentName` for Azure OpenAI Model | String | |
| AZURE_OPENAI_API_VERSION | Default `credential.azureOpenAIApiVersion` for Azure OpenAI Model | String | |
## License
Source code in this repository is made available under the [Apache License Version 2.0](https://github.com/FlowiseAI/Flowise/blob/master/LICENSE.md).
Source code in this repository is made available under the [Apache License Version 2.0](https://github.com/FlowiseAI/Flowise/blob/master/LICENSE.md).
packages/components/nodes/engine/ChatEngine/ContextChatEngine.ts
+4
View File
@@ -10,6 +10,7 @@ import {
} from '../../../src/Interface'
import { Metadata, BaseRetriever, LLM, ContextChatEngine, ChatMessage, NodeWithScore } from 'llamaindex'
import { reformatSourceDocuments } from '../EngineUtils'
import { EvaluationRunTracerLlama } from '../../../evaluation/EvaluationRunTracerLlama'
class ContextChatEngine_LlamaIndex implements INode {
label: string
@@ -93,6 +94,9 @@ class ContextChatEngine_LlamaIndex implements INode {
const chatEngine = new ContextChatEngine({ chatModel: model, retriever: vectorStoreRetriever })
// these are needed for evaluation runs
await EvaluationRunTracerLlama.injectEvaluationMetadata(nodeData, options, chatEngine)
const msgs = (await memory.getChatMessages(this.sessionId, false, prependMessages)) as IMessage[]
for (const message of msgs) {
if (message.type === 'apiMessage') {
packages/components/nodes/engine/ChatEngine/SimpleChatEngine.ts
+4
View File
@@ -9,6 +9,7 @@ import {
IServerSideEventStreamer
} from '../../../src/Interface'
import { LLM, ChatMessage, SimpleChatEngine } from 'llamaindex'
import { EvaluationRunTracerLlama } from '../../../evaluation/EvaluationRunTracerLlama'
class SimpleChatEngine_LlamaIndex implements INode {
label: string
@@ -78,6 +79,9 @@ class SimpleChatEngine_LlamaIndex implements INode {
const chatEngine = new SimpleChatEngine({ llm: model })
// these are needed for evaluation runs
await EvaluationRunTracerLlama.injectEvaluationMetadata(nodeData, options, chatEngine)
const msgs = (await memory.getChatMessages(this.sessionId, false, prependMessages)) as IMessage[]
for (const message of msgs) {
if (message.type === 'apiMessage') {
packages/components/nodes/engine/QueryEngine/QueryEngine.ts
+3
View File
@@ -10,6 +10,7 @@ import {
NodeWithScore
} from 'llamaindex'
import { reformatSourceDocuments } from '../EngineUtils'
import { EvaluationRunTracerLlama } from '../../../evaluation/EvaluationRunTracerLlama'
class QueryEngine_LlamaIndex implements INode {
label: string
@@ -72,6 +73,8 @@ class QueryEngine_LlamaIndex implements INode {
let sourceNodes: NodeWithScore<Metadata>[] = []
let isStreamingStarted = false
await EvaluationRunTracerLlama.injectEvaluationMetadata(nodeData, options, queryEngine)
const shouldStreamResponse = options.shouldStreamResponse
const sseStreamer: IServerSideEventStreamer = options.sseStreamer as IServerSideEventStreamer
const chatId = options.chatId
packages/components/nodes/engine/SubQuestionQueryEngine/SubQuestionQueryEngine.ts
+3
View File
@@ -15,6 +15,7 @@ import {
NodeWithScore
} from 'llamaindex'
import { reformatSourceDocuments } from '../EngineUtils'
import { EvaluationRunTracerLlama } from '../../../evaluation/EvaluationRunTracerLlama'
class SubQuestionQueryEngine_LlamaIndex implements INode {
label: string
@@ -89,6 +90,8 @@ class SubQuestionQueryEngine_LlamaIndex implements INode {
let sourceNodes: NodeWithScore<Metadata>[] = []
let isStreamingStarted = false
await EvaluationRunTracerLlama.injectEvaluationMetadata(nodeData, options, queryEngine)
const shouldStreamResponse = options.shouldStreamResponse
const sseStreamer: IServerSideEventStreamer = options.sseStreamer as IServerSideEventStreamer
const chatId = options.chatId
packages/components/nodes/llms/Azure OpenAI/README.md
+7 -7
View File
@@ -4,13 +4,13 @@ Azure OpenAI LLM integration for Flowise
## 🌱 Env Variables
| Variable | Description | Type | Default |
| ---------------------------- | ----------------------------------------------------------------------------------------------- | ------------------------------------------------ | ----------------------------------- |
| AZURE_OPENAI_API_KEY | Default `credential.azureOpenAIApiKey` for Azure OpenAI LLM | String | |
| AZURE_OPENAI_API_INSTANCE_NAME | Default `credential.azureOpenAIApiInstanceName` for Azure OpenAI LLM | String | |
| AZURE_OPENAI_API_DEPLOYMENT_NAME | Default `credential.azureOpenAIApiDeploymentName` for Azure OpenAI LLM | String | |
| AZURE_OPENAI_API_VERSION | Default `credential.azureOpenAIApiVersion` for Azure OpenAI LLM | String | |
| Variable | Description | Type | Default |
| -------------------------------- | ---------------------------------------------------------------------- | ------ | ------- |
| AZURE_OPENAI_API_KEY | Default `credential.azureOpenAIApiKey` for Azure OpenAI LLM | String | |
| AZURE_OPENAI_API_INSTANCE_NAME | Default `credential.azureOpenAIApiInstanceName` for Azure OpenAI LLM | String | |
| AZURE_OPENAI_API_DEPLOYMENT_NAME | Default `credential.azureOpenAIApiDeploymentName` for Azure OpenAI LLM | String | |
| AZURE_OPENAI_API_VERSION | Default `credential.azureOpenAIApiVersion` for Azure OpenAI LLM | String | |
## License
Source code in this repository is made available under the [Apache License Version 2.0](https://github.com/FlowiseAI/Flowise/blob/master/LICENSE.md).
Source code in this repository is made available under the [Apache License Version 2.0](https://github.com/FlowiseAI/Flowise/blob/master/LICENSE.md).
packages/components/nodes/memory/AgentMemory/AgentMemory.ts
+7 -3
View File
@@ -108,6 +108,7 @@ class AgentMemory_Memory implements INode {
const databaseType = nodeData.inputs?.databaseType as string
const databaseEntities = options.databaseEntities as IDatabaseEntity
const chatflowid = options.chatflowid as string
const orgId = options.orgId as string
const appDataSource = options.appDataSource as DataSource
let additionalConfiguration = {}
@@ -135,7 +136,8 @@ class AgentMemory_Memory implements INode {
threadId,
appDataSource,
databaseEntities,
chatflowid
chatflowid,
orgId
}
const recordManager = new SqliteSaver(args)
return recordManager
@@ -159,7 +161,8 @@ class AgentMemory_Memory implements INode {
threadId,
appDataSource,
databaseEntities,
chatflowid
chatflowid,
orgId
}
const recordManager = new PostgresSaver(args)
return recordManager
@@ -184,7 +187,8 @@ class AgentMemory_Memory implements INode {
threadId,
appDataSource,
databaseEntities,
chatflowid
chatflowid,
orgId
}
const recordManager = new MySQLSaver(args)
return recordManager
packages/components/nodes/memory/AgentMemory/MySQLAgentMemory/MySQLAgentMemory.ts
+3 -1
View File
@@ -65,6 +65,7 @@ class MySQLAgentMemory_Memory implements INode {
const databaseEntities = options.databaseEntities as IDatabaseEntity
const chatflowid = options.chatflowid as string
const appDataSource = options.appDataSource as DataSource
const orgId = options.orgId as string
let additionalConfiguration = {}
if (additionalConfig) {
@@ -102,7 +103,8 @@ class MySQLAgentMemory_Memory implements INode {
threadId,
appDataSource,
databaseEntities,
chatflowid
chatflowid,
orgId
}
const recordManager = new MySQLSaver(args)
return recordManager
packages/components/nodes/memory/AgentMemory/MySQLAgentMemory/mysqlSaver.ts
+1 -1
View File
@@ -242,7 +242,7 @@ export class MySQLSaver extends BaseCheckpointSaver implements MemoryMethods {
}
if (returnBaseMessages) {
return await mapChatMessageToBaseMessage(chatMessage)
return await mapChatMessageToBaseMessage(chatMessage, this.config.orgId)
}
let returnIMessages: IMessage[] = []
packages/components/nodes/memory/AgentMemory/PostgresAgentMemory/PostgresAgentMemory.ts
+3 -1
View File
@@ -65,6 +65,7 @@ class PostgresAgentMemory_Memory implements INode {
const databaseEntities = options.databaseEntities as IDatabaseEntity
const chatflowid = options.chatflowid as string
const appDataSource = options.appDataSource as DataSource
const orgId = options.orgId as string
let additionalConfiguration = {}
if (additionalConfig) {
@@ -101,7 +102,8 @@ class PostgresAgentMemory_Memory implements INode {
threadId,
appDataSource,
databaseEntities,
chatflowid
chatflowid,
orgId
}
const recordManager = new PostgresSaver(args)
return recordManager
packages/components/nodes/memory/AgentMemory/PostgresAgentMemory/pgSaver.ts
+1 -1
View File
@@ -283,7 +283,7 @@ CREATE TABLE IF NOT EXISTS ${tableName} (
}
if (returnBaseMessages) {
return await mapChatMessageToBaseMessage(chatMessage)
return await mapChatMessageToBaseMessage(chatMessage, this.config.orgId)
}
let returnIMessages: IMessage[] = []
packages/components/nodes/memory/AgentMemory/SQLiteAgentMemory/SQLiteAgentMemory.ts
+3 -1
View File
@@ -51,6 +51,7 @@ class SQLiteAgentMemory_Memory implements INode {
const databaseEntities = options.databaseEntities as IDatabaseEntity
const chatflowid = options.chatflowid as string
const appDataSource = options.appDataSource as DataSource
const orgId = options.orgId as string
let additionalConfiguration = {}
if (additionalConfig) {
@@ -76,7 +77,8 @@ class SQLiteAgentMemory_Memory implements INode {
threadId,
appDataSource,
databaseEntities,
chatflowid
chatflowid,
orgId
}
const recordManager = new SqliteSaver(args)
packages/components/nodes/memory/AgentMemory/SQLiteAgentMemory/sqliteSaver.ts
+1 -1
View File
@@ -266,7 +266,7 @@ CREATE TABLE IF NOT EXISTS ${tableName} (
}
if (returnBaseMessages) {
return await mapChatMessageToBaseMessage(chatMessage)
return await mapChatMessageToBaseMessage(chatMessage, this.config.orgId)
}
let returnIMessages: IMessage[] = []
packages/components/nodes/memory/AgentMemory/interface.ts
+1
View File
@@ -9,6 +9,7 @@ export type SaverOptions = {
appDataSource: DataSource
databaseEntities: IDatabaseEntity
chatflowid: string
orgId: string
}
export interface CheckpointTuple {
packages/components/nodes/memory/BufferMemory/BufferMemory.ts
+7 -2
View File
@@ -61,6 +61,7 @@ class BufferMemory_Memory implements INode {
const appDataSource = options.appDataSource as DataSource
const databaseEntities = options.databaseEntities as IDatabaseEntity
const chatflowid = options.chatflowid as string
const orgId = options.orgId as string
return new BufferMemoryExtended({
returnMessages: true,
@@ -68,7 +69,8 @@ class BufferMemory_Memory implements INode {
sessionId,
appDataSource,
databaseEntities,
chatflowid
chatflowid,
orgId
})
}
}
@@ -78,12 +80,14 @@ interface BufferMemoryExtendedInput {
appDataSource: DataSource
databaseEntities: IDatabaseEntity
chatflowid: string
orgId: string
}
class BufferMemoryExtended extends FlowiseMemory implements MemoryMethods {
appDataSource: DataSource
databaseEntities: IDatabaseEntity
chatflowid: string
orgId: string
sessionId = ''
constructor(fields: BufferMemoryInput & BufferMemoryExtendedInput) {
@@ -92,6 +96,7 @@ class BufferMemoryExtended extends FlowiseMemory implements MemoryMethods {
this.appDataSource = fields.appDataSource
this.databaseEntities = fields.databaseEntities
this.chatflowid = fields.chatflowid
this.orgId = fields.orgId
}
async getChatMessages(
@@ -117,7 +122,7 @@ class BufferMemoryExtended extends FlowiseMemory implements MemoryMethods {
}
if (returnBaseMessages) {
return await mapChatMessageToBaseMessage(chatMessage)
return await mapChatMessageToBaseMessage(chatMessage, this.orgId)
}
let returnIMessages: IMessage[] = []
packages/components/nodes/memory/BufferWindowMemory/BufferWindowMemory.ts
+7 -2
View File
@@ -69,6 +69,7 @@ class BufferWindowMemory_Memory implements INode {
const appDataSource = options.appDataSource as DataSource
const databaseEntities = options.databaseEntities as IDatabaseEntity
const chatflowid = options.chatflowid as string
const orgId = options.orgId as string
const obj: Partial<BufferWindowMemoryInput> & BufferMemoryExtendedInput = {
returnMessages: true,
@@ -77,7 +78,8 @@ class BufferWindowMemory_Memory implements INode {
k: parseInt(k, 10),
appDataSource,
databaseEntities,
chatflowid
chatflowid,
orgId
}
return new BufferWindowMemoryExtended(obj)
@@ -89,12 +91,14 @@ interface BufferMemoryExtendedInput {
appDataSource: DataSource
databaseEntities: IDatabaseEntity
chatflowid: string
orgId: string
}
class BufferWindowMemoryExtended extends FlowiseWindowMemory implements MemoryMethods {
appDataSource: DataSource
databaseEntities: IDatabaseEntity
chatflowid: string
orgId: string
sessionId = ''
constructor(fields: BufferWindowMemoryInput & BufferMemoryExtendedInput) {
@@ -103,6 +107,7 @@ class BufferWindowMemoryExtended extends FlowiseWindowMemory implements MemoryMe
this.appDataSource = fields.appDataSource
this.databaseEntities = fields.databaseEntities
this.chatflowid = fields.chatflowid
this.orgId = fields.orgId
}
async getChatMessages(
@@ -134,7 +139,7 @@ class BufferWindowMemoryExtended extends FlowiseWindowMemory implements MemoryMe
}
if (returnBaseMessages) {
return await mapChatMessageToBaseMessage(chatMessage)
return await mapChatMessageToBaseMessage(chatMessage, this.orgId)
}
let returnIMessages: IMessage[] = []
packages/components/nodes/memory/ConversationSummaryBufferMemory/ConversationSummaryBufferMemory.ts
+7 -2
View File
@@ -78,6 +78,7 @@ class ConversationSummaryBufferMemory_Memory implements INode {
const appDataSource = options.appDataSource as DataSource
const databaseEntities = options.databaseEntities as IDatabaseEntity
const chatflowid = options.chatflowid as string
const orgId = options.orgId as string
const obj: ConversationSummaryBufferMemoryInput & BufferMemoryExtendedInput = {
llm: model,
@@ -87,7 +88,8 @@ class ConversationSummaryBufferMemory_Memory implements INode {
returnMessages: true,
appDataSource,
databaseEntities,
chatflowid
chatflowid,
orgId
}
return new ConversationSummaryBufferMemoryExtended(obj)
@@ -99,12 +101,14 @@ interface BufferMemoryExtendedInput {
appDataSource: DataSource
databaseEntities: IDatabaseEntity
chatflowid: string
orgId: string
}
class ConversationSummaryBufferMemoryExtended extends FlowiseSummaryBufferMemory implements MemoryMethods {
appDataSource: DataSource
databaseEntities: IDatabaseEntity
chatflowid: string
orgId: string
sessionId = ''
constructor(fields: ConversationSummaryBufferMemoryInput & BufferMemoryExtendedInput) {
@@ -113,6 +117,7 @@ class ConversationSummaryBufferMemoryExtended extends FlowiseSummaryBufferMemory
this.appDataSource = fields.appDataSource
this.databaseEntities = fields.databaseEntities
this.chatflowid = fields.chatflowid
this.orgId = fields.orgId
}
async getChatMessages(
@@ -137,7 +142,7 @@ class ConversationSummaryBufferMemoryExtended extends FlowiseSummaryBufferMemory
chatMessage.unshift(...prependMessages)
}
let baseMessages = await mapChatMessageToBaseMessage(chatMessage)
let baseMessages = await mapChatMessageToBaseMessage(chatMessage, this.orgId)
// Prune baseMessages if it exceeds max token limit
if (this.movingSummaryBuffer) {
packages/components/nodes/memory/ConversationSummaryMemory/ConversationSummaryMemory.ts
+7 -2
View File
@@ -69,6 +69,7 @@ class ConversationSummaryMemory_Memory implements INode {
const appDataSource = options.appDataSource as DataSource
const databaseEntities = options.databaseEntities as IDatabaseEntity
const chatflowid = options.chatflowid as string
const orgId = options.orgId as string
const obj: ConversationSummaryMemoryInput & BufferMemoryExtendedInput = {
llm: model,
@@ -77,7 +78,8 @@ class ConversationSummaryMemory_Memory implements INode {
sessionId,
appDataSource,
databaseEntities,
chatflowid
chatflowid,
orgId
}
return new ConversationSummaryMemoryExtended(obj)
@@ -89,12 +91,14 @@ interface BufferMemoryExtendedInput {
appDataSource: DataSource
databaseEntities: IDatabaseEntity
chatflowid: string
orgId: string
}
class ConversationSummaryMemoryExtended extends FlowiseSummaryMemory implements MemoryMethods {
appDataSource: DataSource
databaseEntities: IDatabaseEntity
chatflowid: string
orgId: string
sessionId = ''
constructor(fields: ConversationSummaryMemoryInput & BufferMemoryExtendedInput) {
@@ -103,6 +107,7 @@ class ConversationSummaryMemoryExtended extends FlowiseSummaryMemory implements
this.appDataSource = fields.appDataSource
this.databaseEntities = fields.databaseEntities
this.chatflowid = fields.chatflowid
this.orgId = fields.orgId
}
async getChatMessages(
@@ -128,7 +133,7 @@ class ConversationSummaryMemoryExtended extends FlowiseSummaryMemory implements
chatMessage.unshift(...prependMessages)
}
const baseMessages = await mapChatMessageToBaseMessage(chatMessage)
const baseMessages = await mapChatMessageToBaseMessage(chatMessage, this.orgId)
// Get summary
if (this.llm && typeof this.llm !== 'string') {
packages/components/nodes/memory/DynamoDb/DynamoDb.ts
+8 -2
View File
@@ -125,6 +125,8 @@ const initializeDynamoDB = async (nodeData: INodeData, options: ICommonObject):
config
})
const orgId = options.orgId as string
const memory = new BufferMemoryExtended({
memoryKey: memoryKey ?? 'chat_history',
chatHistory: dynamoDb,
@@ -132,7 +134,8 @@ const initializeDynamoDB = async (nodeData: INodeData, options: ICommonObject):
dynamodbClient: client,
tableName,
partitionKey,
dynamoKey: { [partitionKey]: { S: sessionId } }
dynamoKey: { [partitionKey]: { S: sessionId } },
orgId
})
return memory
}
@@ -143,6 +146,7 @@ interface BufferMemoryExtendedInput {
tableName: string
partitionKey: string
dynamoKey: Record<string, AttributeValue>
orgId: string
}
interface DynamoDBSerializedChatMessage {
@@ -165,6 +169,7 @@ class BufferMemoryExtended extends FlowiseMemory implements MemoryMethods {
private dynamoKey: Record<string, AttributeValue>
private messageAttributeName: string
sessionId = ''
orgId = ''
dynamodbClient: DynamoDBClient
constructor(fields: BufferMemoryInput & BufferMemoryExtendedInput) {
@@ -174,6 +179,7 @@ class BufferMemoryExtended extends FlowiseMemory implements MemoryMethods {
this.tableName = fields.tableName
this.partitionKey = fields.partitionKey
this.dynamoKey = fields.dynamoKey
this.orgId = fields.orgId
}
overrideDynamoKey(overrideSessionId = '') {
@@ -260,7 +266,7 @@ class BufferMemoryExtended extends FlowiseMemory implements MemoryMethods {
.filter((x): x is StoredMessage => x.type !== undefined && x.data.content !== undefined)
const baseMessages = messages.map(mapStoredMessageToChatMessage)
if (prependMessages?.length) {
baseMessages.unshift(...(await mapChatMessageToBaseMessage(prependMessages)))
baseMessages.unshift(...(await mapChatMessageToBaseMessage(prependMessages, this.orgId)))
}
return returnBaseMessages ? baseMessages : convertBaseMessagetoIMessage(baseMessages)
}
packages/components/nodes/memory/Mem0/Mem0.ts
+7 -2
View File
@@ -151,6 +151,7 @@ class Mem0_Memory implements INode {
const initializeMem0 = async (nodeData: INodeData, options: ICommonObject): Promise<BaseMem0Memory> => {
const initialUserId = nodeData.inputs?.user_id as string
const useFlowiseChatId = nodeData.inputs?.useFlowiseChatId as boolean
const orgId = options.orgId as string
if (!useFlowiseChatId && !initialUserId) {
throw new Error('User ID field cannot be empty when "Use Flowise Chat ID" is OFF.')
@@ -198,7 +199,8 @@ const initializeMem0 = async (nodeData: INodeData, options: ICommonObject): Prom
databaseEntities: options.databaseEntities as IDatabaseEntity,
chatflowid: options.chatflowid as string,
searchOnly: (nodeData.inputs?.searchOnly as boolean) || false,
useFlowiseChatId: useFlowiseChatId
useFlowiseChatId: useFlowiseChatId,
orgId: orgId
}
return new Mem0MemoryExtended(obj)
@@ -207,11 +209,13 @@ const initializeMem0 = async (nodeData: INodeData, options: ICommonObject): Prom
interface Mem0MemoryExtendedInput extends Mem0MemoryInput {
memoryOptions?: MemoryOptions | SearchOptions
useFlowiseChatId: boolean
orgId: string
}
class Mem0MemoryExtended extends BaseMem0Memory implements MemoryMethods {
initialUserId: string
userId: string
orgId: string
memoryKey: string
inputKey: string
appDataSource: DataSource
@@ -233,6 +237,7 @@ class Mem0MemoryExtended extends BaseMem0Memory implements MemoryMethods {
this.chatflowid = fields.chatflowid
this.searchOnly = fields.searchOnly
this.useFlowiseChatId = fields.useFlowiseChatId
this.orgId = fields.orgId
}
// Selects Mem0 user_id based on toggle state (Flowise chat ID or input field)
@@ -337,7 +342,7 @@ class Mem0MemoryExtended extends BaseMem0Memory implements MemoryMethods {
console.warn('Mem0 history is not a string, cannot prepend directly.')
}
return await mapChatMessageToBaseMessage(chatMessage)
return await mapChatMessageToBaseMessage(chatMessage, this.orgId)
}
return returnIMessages
packages/components/nodes/memory/MongoDBMemory/MongoDBMemory.ts
+7 -1
View File
@@ -88,9 +88,12 @@ const initializeMongoDB = async (nodeData: INodeData, options: ICommonObject): P
const mongoDBConnectUrl = getCredentialParam('mongoDBConnectUrl', credentialData, nodeData)
const driverInfo = { name: 'Flowise', version: (await getVersion()).version }
const orgId = options.orgId as string
return new BufferMemoryExtended({
memoryKey: memoryKey ?? 'chat_history',
sessionId,
orgId,
mongoConnection: {
databaseName,
collectionName,
@@ -102,6 +105,7 @@ const initializeMongoDB = async (nodeData: INodeData, options: ICommonObject): P
interface BufferMemoryExtendedInput {
sessionId: string
orgId: string
mongoConnection: {
databaseName: string
collectionName: string
@@ -112,6 +116,7 @@ interface BufferMemoryExtendedInput {
class BufferMemoryExtended extends FlowiseMemory implements MemoryMethods {
sessionId = ''
orgId = ''
mongoConnection: {
databaseName: string
collectionName: string
@@ -122,6 +127,7 @@ class BufferMemoryExtended extends FlowiseMemory implements MemoryMethods {
constructor(fields: BufferMemoryInput & BufferMemoryExtendedInput) {
super(fields)
this.sessionId = fields.sessionId
this.orgId = fields.orgId
this.mongoConnection = fields.mongoConnection
}
@@ -138,7 +144,7 @@ class BufferMemoryExtended extends FlowiseMemory implements MemoryMethods {
const messages = document?.messages || []
const baseMessages = messages.map(mapStoredMessageToChatMessage)
if (prependMessages?.length) {
baseMessages.unshift(...(await mapChatMessageToBaseMessage(prependMessages)))
baseMessages.unshift(...(await mapChatMessageToBaseMessage(prependMessages, this.orgId)))
}
await client.close()
packages/components/nodes/memory/RedisBackedChatMemory/RedisBackedChatMemory.ts
+7 -2
View File
@@ -88,6 +88,7 @@ const initializeRedis = async (nodeData: INodeData, options: ICommonObject): Pro
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
const redisUrl = getCredentialParam('redisUrl', credentialData, nodeData)
const orgId = options.orgId as string
const redisOptions = redisUrl
? redisUrl
@@ -104,7 +105,8 @@ const initializeRedis = async (nodeData: INodeData, options: ICommonObject): Pro
sessionId,
windowSize,
sessionTTL,
redisOptions
redisOptions,
orgId
})
return memory
@@ -114,11 +116,13 @@ interface BufferMemoryExtendedInput {
sessionId: string
windowSize?: number
sessionTTL?: number
orgId: string
redisOptions: RedisOptions | string
}
class BufferMemoryExtended extends FlowiseMemory implements MemoryMethods {
sessionId = ''
orgId = ''
windowSize?: number
sessionTTL?: number
redisOptions: RedisOptions | string
@@ -128,6 +132,7 @@ class BufferMemoryExtended extends FlowiseMemory implements MemoryMethods {
this.sessionId = fields.sessionId
this.windowSize = fields.windowSize
this.sessionTTL = fields.sessionTTL
this.orgId = fields.orgId
this.redisOptions = fields.redisOptions
}
@@ -165,7 +170,7 @@ class BufferMemoryExtended extends FlowiseMemory implements MemoryMethods {
const orderedMessages = rawStoredMessages.reverse().map((message) => JSON.parse(message))
const baseMessages = orderedMessages.map(mapStoredMessageToChatMessage)
if (prependMessages?.length) {
baseMessages.unshift(...(await mapChatMessageToBaseMessage(prependMessages)))
baseMessages.unshift(...(await mapChatMessageToBaseMessage(prependMessages, this.orgId)))
}
return returnBaseMessages ? baseMessages : convertBaseMessagetoIMessage(baseMessages)
})
packages/components/nodes/memory/UpstashRedisBackedChatMemory/UpstashRedisBackedChatMemory.ts
+7 -3
View File
@@ -100,13 +100,14 @@ const initalizeUpstashRedis = async (nodeData: INodeData, options: ICommonObject
sessionTTL,
client
})
const orgId = options.orgId as string
const memory = new BufferMemoryExtended({
memoryKey: memoryKey ?? 'chat_history',
chatHistory: redisChatMessageHistory,
sessionId,
sessionTTL,
redisClient: client
redisClient: client,
orgId
})
return memory
@@ -115,11 +116,13 @@ const initalizeUpstashRedis = async (nodeData: INodeData, options: ICommonObject
interface BufferMemoryExtendedInput {
redisClient: Redis
sessionId: string
orgId: string
sessionTTL?: number
}
class BufferMemoryExtended extends FlowiseMemory implements MemoryMethods {
sessionId = ''
orgId = ''
redisClient: Redis
sessionTTL?: number
@@ -128,6 +131,7 @@ class BufferMemoryExtended extends FlowiseMemory implements MemoryMethods {
this.sessionId = fields.sessionId
this.redisClient = fields.redisClient
this.sessionTTL = fields.sessionTTL
this.orgId = fields.orgId
}
async getChatMessages(
@@ -143,7 +147,7 @@ class BufferMemoryExtended extends FlowiseMemory implements MemoryMethods {
const previousMessages = orderedMessages.filter((x): x is StoredMessage => x.type !== undefined && x.data.content !== undefined)
const baseMessages = previousMessages.map(mapStoredMessageToChatMessage)
if (prependMessages?.length) {
baseMessages.unshift(...(await mapChatMessageToBaseMessage(prependMessages)))
baseMessages.unshift(...(await mapChatMessageToBaseMessage(prependMessages, this.orgId)))
}
return returnBaseMessages ? baseMessages : convertBaseMessagetoIMessage(baseMessages)
}
packages/components/nodes/memory/ZepMemory/ZepMemory.ts
+6 -1
View File
@@ -119,6 +119,7 @@ const initializeZep = async (nodeData: INodeData, options: ICommonObject): Promi
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
const apiKey = getCredentialParam('apiKey', credentialData, nodeData)
const orgId = options.orgId as string
const obj: ZepMemoryInput & ZepMemoryExtendedInput = {
baseURL,
aiPrefix,
@@ -127,6 +128,7 @@ const initializeZep = async (nodeData: INodeData, options: ICommonObject): Promi
memoryKey,
inputKey,
sessionId,
orgId,
k: k ? parseInt(k, 10) : undefined
}
if (apiKey) obj.apiKey = apiKey
@@ -136,14 +138,17 @@ const initializeZep = async (nodeData: INodeData, options: ICommonObject): Promi
interface ZepMemoryExtendedInput {
k?: number
orgId: string
}
class ZepMemoryExtended extends ZepMemory implements MemoryMethods {
lastN?: number
orgId = ''
constructor(fields: ZepMemoryInput & ZepMemoryExtendedInput) {
super(fields)
this.lastN = fields.k
this.orgId = fields.orgId
}
async loadMemoryVariables(values: InputValues, overrideSessionId = ''): Promise<MemoryVariables> {
@@ -176,7 +181,7 @@ class ZepMemoryExtended extends ZepMemory implements MemoryMethods {
const memoryVariables = await this.loadMemoryVariables({}, id)
const baseMessages = memoryVariables[this.memoryKey]
if (prependMessages?.length) {
baseMessages.unshift(...(await mapChatMessageToBaseMessage(prependMessages)))
baseMessages.unshift(...(await mapChatMessageToBaseMessage(prependMessages, this.orgId)))
}
return returnBaseMessages ? baseMessages : convertBaseMessagetoIMessage(baseMessages)
}
packages/components/nodes/memory/ZepMemoryCloud/ZepMemoryCloud.ts
+7 -2
View File
@@ -113,6 +113,7 @@ const initializeZep = async (nodeData: INodeData, options: ICommonObject): Promi
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
const apiKey = getCredentialParam('apiKey', credentialData, nodeData)
const orgId = options.orgId as string
const obj: ZepMemoryInput & ZepMemoryExtendedInput = {
apiKey,
aiPrefix,
@@ -121,7 +122,8 @@ const initializeZep = async (nodeData: INodeData, options: ICommonObject): Promi
sessionId,
inputKey,
memoryType: memoryType,
returnMessages: true
returnMessages: true,
orgId
}
return new ZepMemoryExtended(obj)
@@ -129,14 +131,17 @@ const initializeZep = async (nodeData: INodeData, options: ICommonObject): Promi
interface ZepMemoryExtendedInput {
memoryType?: 'perpetual' | 'message_window'
orgId: string
}
class ZepMemoryExtended extends ZepMemory implements MemoryMethods {
memoryType: 'perpetual' | 'message_window'
orgId: string
constructor(fields: ZepMemoryInput & ZepMemoryExtendedInput) {
super(fields)
this.memoryType = fields.memoryType ?? 'perpetual'
this.orgId = fields.orgId
}
async loadMemoryVariables(values: InputValues, overrideSessionId = ''): Promise<MemoryVariables> {
@@ -169,7 +174,7 @@ class ZepMemoryExtended extends ZepMemory implements MemoryMethods {
const memoryVariables = await this.loadMemoryVariables({}, id)
const baseMessages = memoryVariables[this.memoryKey]
if (prependMessages?.length) {
baseMessages.unshift(...(await mapChatMessageToBaseMessage(prependMessages)))
baseMessages.unshift(...(await mapChatMessageToBaseMessage(prependMessages, this.orgId)))
}
return returnBaseMessages ? baseMessages : convertBaseMessagetoIMessage(baseMessages)
}
packages/components/nodes/multiagents/Worker/Worker.ts
+1 -1
View File
@@ -233,7 +233,7 @@ async function createAgent(
sessionId: flowObj?.sessionId,
chatId: flowObj?.chatId,
input: flowObj?.input,
verbose: process.env.DEBUG === 'true',
verbose: process.env.DEBUG === 'true' ? true : false,
maxIterations: maxIterations ? parseFloat(maxIterations) : undefined
})
return executor
packages/components/nodes/prompts/ChatPromptTemplate/ChatPromptTemplate.ts
+1 -1
View File
@@ -120,7 +120,7 @@ class ChatPromptTemplate_Prompts implements INode {
) {
const appDataSource = options.appDataSource as DataSource
const databaseEntities = options.databaseEntities as IDatabaseEntity
const vm = await getVM(appDataSource, databaseEntities, nodeData, {})
const vm = await getVM(appDataSource, databaseEntities, nodeData, options, {})
try {
const response = await vm.run(`module.exports = async function() {${messageHistoryCode}}()`, __dirname)
if (!Array.isArray(response)) throw new Error('Returned message history must be an array')
packages/components/nodes/sequentialagents/Agent/Agent.ts
+3 -3
View File
@@ -680,7 +680,7 @@ async function createAgent(
sessionId: flowObj?.sessionId,
chatId: flowObj?.chatId,
input: flowObj?.input,
verbose: process.env.DEBUG === 'true',
verbose: process.env.DEBUG === 'true' ? true : false,
maxIterations: maxIterations ? parseFloat(maxIterations) : undefined
})
return executor
@@ -877,7 +877,7 @@ const getReturnOutput = async (nodeData: INodeData, input: string, options: ICom
const updateStateMemory = nodeData.inputs?.updateStateMemory as string
const selectedTab = tabIdentifier ? tabIdentifier.split(`_${nodeData.id}`)[0] : 'updateStateMemoryUI'
const variables = await getVars(appDataSource, databaseEntities, nodeData)
const variables = await getVars(appDataSource, databaseEntities, nodeData, options)
const flow = {
chatflowId: options.chatflowid,
@@ -930,7 +930,7 @@ const getReturnOutput = async (nodeData: INodeData, input: string, options: ICom
throw new Error(e)
}
} else if (selectedTab === 'updateStateMemoryCode' && updateStateMemoryCode) {
const vm = await getVM(appDataSource, databaseEntities, nodeData, flow)
const vm = await getVM(appDataSource, databaseEntities, nodeData, options, flow)
try {
const response = await vm.run(`module.exports = async function() {${updateStateMemoryCode}}()`, __dirname)
if (typeof response !== 'object') throw new Error('Return output must be an object')
packages/components/nodes/sequentialagents/Condition/Condition.ts
+2 -2
View File
@@ -267,7 +267,7 @@ const runCondition = async (nodeData: INodeData, input: string, options: ICommon
const tabIdentifier = nodeData.inputs?.[`${TAB_IDENTIFIER}_${nodeData.id}`] as string
const selectedTab = tabIdentifier ? tabIdentifier.split(`_${nodeData.id}`)[0] : 'conditionUI'
const variables = await getVars(appDataSource, databaseEntities, nodeData)
const variables = await getVars(appDataSource, databaseEntities, nodeData, options)
const flow = {
chatflowId: options.chatflowid,
@@ -279,7 +279,7 @@ const runCondition = async (nodeData: INodeData, input: string, options: ICommon
}
if (selectedTab === 'conditionFunction' && conditionFunction) {
const vm = await getVM(appDataSource, databaseEntities, nodeData, flow)
const vm = await getVM(appDataSource, databaseEntities, nodeData, options, flow)
try {
const response = await vm.run(`module.exports = async function() {${conditionFunction}}()`, __dirname)
if (typeof response !== 'string') throw new Error('Condition function must return a string')
packages/components/nodes/sequentialagents/ConditionAgent/ConditionAgent.ts
+2 -2
View File
@@ -540,7 +540,7 @@ const runCondition = async (
result = { ...jsonResult, additional_kwargs: { nodeId: nodeData.id } }
}
const variables = await getVars(appDataSource, databaseEntities, nodeData)
const variables = await getVars(appDataSource, databaseEntities, nodeData, options)
const flow = {
chatflowId: options.chatflowid,
@@ -553,7 +553,7 @@ const runCondition = async (
}
if (selectedTab === 'conditionFunction' && conditionFunction) {
const vm = await getVM(appDataSource, databaseEntities, nodeData, flow)
const vm = await getVM(appDataSource, databaseEntities, nodeData, options, flow)
try {
const response = await vm.run(`module.exports = async function() {${conditionFunction}}()`, __dirname)
if (typeof response !== 'string') throw new Error('Condition function must return a string')
packages/components/nodes/sequentialagents/CustomFunction/CustomFunction.ts
+1 -1
View File
@@ -102,7 +102,7 @@ class CustomFunction_SeqAgents implements INode {
if (!sequentialNodes || !sequentialNodes.length) throw new Error('Custom function must have a predecessor!')
const executeFunc = async (state: ISeqAgentsState) => {
const variables = await getVars(appDataSource, databaseEntities, nodeData)
const variables = await getVars(appDataSource, databaseEntities, nodeData, options)
const flow = {
chatflowId: options.chatflowid,
sessionId: options.sessionId,
packages/components/nodes/sequentialagents/ExecuteFlow/ExecuteFlow.ts
+5 -4
View File
@@ -141,7 +141,8 @@ class ExecuteFlow_SeqAgents implements INode {
return returnData
}
const chatflows = await appDataSource.getRepository(databaseEntities['ChatFlow']).find()
const searchOptions = options.searchOptions || {}
const chatflows = await appDataSource.getRepository(databaseEntities['ChatFlow']).findBy(searchOptions)
for (let i = 0; i < chatflows.length; i += 1) {
const data = {
@@ -189,7 +190,7 @@ class ExecuteFlow_SeqAgents implements INode {
const chatId = options.chatId
const executeFunc = async (state: ISeqAgentsState) => {
const variables = await getVars(appDataSource, databaseEntities, nodeData)
const variables = await getVars(appDataSource, databaseEntities, nodeData, options)
let flowInput = ''
if (seqExecuteFlowInput === 'userQuestion') {
@@ -223,7 +224,7 @@ class ExecuteFlow_SeqAgents implements INode {
}
}
const options = {
const callOptions = {
method: 'POST',
headers: {
'Content-Type': 'application/json',
@@ -234,7 +235,7 @@ class ExecuteFlow_SeqAgents implements INode {
let sandbox: ICommonObject = {
$input: flowInput,
$callOptions: options,
$callOptions: callOptions,
$callBody: body,
util: undefined,
Symbol: undefined,
packages/components/nodes/sequentialagents/LLMNode/LLMNode.ts
+2 -2
View File
@@ -668,7 +668,7 @@ const getReturnOutput = async (nodeData: INodeData, input: string, options: ICom
const updateStateMemory = nodeData.inputs?.updateStateMemory as string
const selectedTab = tabIdentifier ? tabIdentifier.split(`_${nodeData.id}`)[0] : 'updateStateMemoryUI'
const variables = await getVars(appDataSource, databaseEntities, nodeData)
const variables = await getVars(appDataSource, databaseEntities, nodeData, options)
const flow = {
chatflowId: options.chatflowid,
@@ -721,7 +721,7 @@ const getReturnOutput = async (nodeData: INodeData, input: string, options: ICom
throw new Error(e)
}
} else if (selectedTab === 'updateStateMemoryCode' && updateStateMemoryCode) {
const vm = await getVM(appDataSource, databaseEntities, nodeData, flow)
const vm = await getVM(appDataSource, databaseEntities, nodeData, options, flow)
try {
const response = await vm.run(`module.exports = async function() {${updateStateMemoryCode}}()`, __dirname)
if (typeof response !== 'object') throw new Error('Return output must be an object')
packages/components/nodes/sequentialagents/State/State.ts
+1 -1
View File
@@ -190,7 +190,7 @@ class State_SeqAgents implements INode {
throw new Error(e)
}
} else if (selectedTab === 'stateMemoryCode' && stateMemoryCode) {
const variables = await getVars(appDataSource, databaseEntities, nodeData)
const variables = await getVars(appDataSource, databaseEntities, nodeData, options)
const flow = {
chatflowId: options.chatflowid,
sessionId: options.sessionId,
packages/components/nodes/sequentialagents/ToolNode/ToolNode.ts
+2 -2
View File
@@ -498,7 +498,7 @@ const getReturnOutput = async (
const updateStateMemory = nodeData.inputs?.updateStateMemory as string
const selectedTab = tabIdentifier ? tabIdentifier.split(`_${nodeData.id}`)[0] : 'updateStateMemoryUI'
const variables = await getVars(appDataSource, databaseEntities, nodeData)
const variables = await getVars(appDataSource, databaseEntities, nodeData, options)
const reformattedOutput = outputs.map((output) => {
return {
@@ -561,7 +561,7 @@ const getReturnOutput = async (
throw new Error(e)
}
} else if (selectedTab === 'updateStateMemoryCode' && updateStateMemoryCode) {
const vm = await getVM(appDataSource, databaseEntities, nodeData, flow)
const vm = await getVM(appDataSource, databaseEntities, nodeData, options, flow)
try {
const response = await vm.run(`module.exports = async function() {${updateStateMemoryCode}}()`, __dirname)
if (typeof response !== 'object') throw new Error('Return output must be an object')
packages/components/nodes/sequentialagents/commonUtils.ts
+9 -3
View File
@@ -150,8 +150,14 @@ export const processImageMessage = async (llm: BaseChatModel, nodeData: INodeDat
return multiModalMessageContent
}
export const getVM = async (appDataSource: DataSource, databaseEntities: IDatabaseEntity, nodeData: INodeData, flow: ICommonObject) => {
const variables = await getVars(appDataSource, databaseEntities, nodeData)
export const getVM = async (
appDataSource: DataSource,
databaseEntities: IDatabaseEntity,
nodeData: INodeData,
options: ICommonObject,
flow: ICommonObject
) => {
const variables = await getVars(appDataSource, databaseEntities, nodeData, options)
let sandbox: any = {
util: undefined,
@@ -420,7 +426,7 @@ export const checkMessageHistory = async (
if (messageHistory) {
const appDataSource = options.appDataSource as DataSource
const databaseEntities = options.databaseEntities as IDatabaseEntity
const vm = await getVM(appDataSource, databaseEntities, nodeData, {})
const vm = await getVM(appDataSource, databaseEntities, nodeData, options, {})
try {
const response = await vm.run(`module.exports = async function() {${messageHistory}}()`, __dirname)
if (!Array.isArray(response)) throw new Error('Returned message history must be an array')
packages/components/nodes/tools/ChainTool/core.ts
+29 -5
View File
@@ -14,17 +14,41 @@ export class ChainTool extends DynamicTool {
super({
...rest,
func: async (input, runManager) => {
const childManagers = runManager?.getChild()
const handlers = childManagers?.handlers?.filter((handler) => !(handler instanceof CustomChainHandler)) || []
if (childManagers) childManagers.handlers = handlers
// prevent sending SSE events of the sub-chain
const sseStreamer = runManager?.handlers.find((handler) => handler instanceof CustomChainHandler)?.sseStreamer
if (runManager) {
const callbacks = runManager.handlers
for (let i = 0; i < callbacks.length; i += 1) {
if (callbacks[i] instanceof CustomChainHandler) {
;(callbacks[i] as any).sseStreamer = undefined
}
}
}
if ((chain as any).prompt && (chain as any).prompt.promptValues) {
const promptValues = handleEscapeCharacters((chain as any).prompt.promptValues, true)
const values = await chain.call(promptValues, childManagers)
const values = await chain.call(promptValues, runManager?.getChild())
if (runManager && sseStreamer) {
const callbacks = runManager.handlers
for (let i = 0; i < callbacks.length; i += 1) {
if (callbacks[i] instanceof CustomChainHandler) {
;(callbacks[i] as any).sseStreamer = sseStreamer
}
}
}
return values?.text
}
const values = chain.run(input, childManagers)
const values = chain.run(input, runManager?.getChild())
if (runManager && sseStreamer) {
const callbacks = runManager.handlers
for (let i = 0; i < callbacks.length; i += 1) {
if (callbacks[i] instanceof CustomChainHandler) {
;(callbacks[i] as any).sseStreamer = sseStreamer
}
}
}
return values
}
})
packages/components/nodes/tools/ChatflowTool/ChatflowTool.ts
+2 -1
View File
@@ -122,7 +122,8 @@ class ChatflowTool_Tools implements INode {
return returnData
}
const chatflows = await appDataSource.getRepository(databaseEntities['ChatFlow']).find()
const searchOptions = options.searchOptions || {}
const chatflows = await appDataSource.getRepository(databaseEntities['ChatFlow']).findBy(searchOptions)
for (let i = 0; i < chatflows.length; i += 1) {
const data = {
packages/components/nodes/tools/CodeInterpreterE2B/CodeInterpreterE2B.ts
+16 -5
View File
@@ -80,7 +80,8 @@ class Code_Interpreter_Tools implements INode {
schema: z.object({
input: z.string().describe('Python code to be executed in the sandbox environment')
}),
chatflowid: options.chatflowid
chatflowid: options.chatflowid,
orgId: options.orgId
})
}
}
@@ -92,6 +93,7 @@ type E2BToolInput = {
apiKey: string
schema: any
chatflowid: string
orgId: string
templateCodeInterpreterE2B?: string
domainCodeInterpreterE2B?: string
}
@@ -113,6 +115,8 @@ export class E2BTool extends StructuredTool {
chatflowid: string
orgId: string
flowObj: ICommonObject
templateCodeInterpreterE2B?: string
@@ -125,6 +129,7 @@ export class E2BTool extends StructuredTool {
this.apiKey = options.apiKey
this.schema = options.schema
this.chatflowid = options.chatflowid
this.orgId = options.orgId
this.templateCodeInterpreterE2B = options.templateCodeInterpreterE2B
this.domainCodeInterpreterE2B = options.domainCodeInterpreterE2B
}
@@ -136,6 +141,7 @@ export class E2BTool extends StructuredTool {
apiKey: options.apiKey,
schema: options.schema,
chatflowid: options.chatflowid,
orgId: options.orgId,
templateCodeInterpreterE2B: options.templateCodeInterpreterE2B,
domainCodeInterpreterE2B: options.domainCodeInterpreterE2B
})
@@ -212,28 +218,33 @@ export class E2BTool extends StructuredTool {
const filename = `artifact_${Date.now()}.png`
const res = await addSingleFileToStorage(
// Don't check storage usage because this is incoming file, and if we throw error, agent will keep on retrying
const { path } = await addSingleFileToStorage(
'image/png',
pngData,
filename,
this.orgId,
this.chatflowid,
flowConfig!.chatId as string
)
artifacts.push({ type: 'png', data: res })
artifacts.push({ type: 'png', data: path })
} else if (key === 'jpeg') {
//@ts-ignore
const jpegData = Buffer.from(result.jpeg, 'base64')
const filename = `artifact_${Date.now()}.jpg`
const res = await addSingleFileToStorage(
const { path } = await addSingleFileToStorage(
'image/jpg',
jpegData,
filename,
this.orgId,
this.chatflowid,
flowConfig!.chatId as string
)
artifacts.push({ type: 'jpeg', data: res })
artifacts.push({ type: 'jpeg', data: path })
} else if (key === 'html' || key === 'markdown' || key === 'latex' || key === 'json' || key === 'javascript') {
artifacts.push({ type: key, data: (result as any)[key] })
} //TODO: support for pdf
packages/components/nodes/tools/CustomTool/CustomTool.ts
+3 -2
View File
@@ -77,7 +77,8 @@ class CustomTool_Tools implements INode {
return returnData
}
const tools = await appDataSource.getRepository(databaseEntities['Tool']).find()
const searchOptions = options.searchOptions || {}
const tools = await appDataSource.getRepository(databaseEntities['Tool']).findBy(searchOptions)
for (let i = 0; i < tools.length; i += 1) {
const data = {
@@ -122,7 +123,7 @@ class CustomTool_Tools implements INode {
obj.schema = zodSchemaFunction(z)
}
const variables = await getVars(appDataSource, databaseEntities, nodeData)
const variables = await getVars(appDataSource, databaseEntities, nodeData, options)
const flow = { chatflowId: options.chatflowid }
packages/components/nodes/tools/OpenAPIToolkit/OpenAPIToolkit.ts
+3 -2
View File
@@ -85,8 +85,9 @@ class OpenAPIToolkit_Tools implements INode {
let data
if (yamlFileBase64.startsWith('FILE-STORAGE::')) {
const file = yamlFileBase64.replace('FILE-STORAGE::', '')
const orgId = options.orgId
const chatflowid = options.chatflowid
const fileData = await getFileFromStorage(file, chatflowid)
const fileData = await getFileFromStorage(file, orgId, chatflowid)
const utf8String = fileData.toString('utf-8')
data = load(utf8String)
@@ -110,7 +111,7 @@ class OpenAPIToolkit_Tools implements INode {
const appDataSource = options.appDataSource as DataSource
const databaseEntities = options.databaseEntities as IDatabaseEntity
const variables = await getVars(appDataSource, databaseEntities, nodeData)
const variables = await getVars(appDataSource, databaseEntities, nodeData, options)
const flow = { chatflowId: options.chatflowid }
packages/components/nodes/utilities/CustomFunction/CustomFunction.ts
+1 -1
View File
@@ -83,7 +83,7 @@ class CustomFunction_Utilities implements INode {
const databaseEntities = options.databaseEntities as IDatabaseEntity
const tools = Object.fromEntries((flatten(nodeData.inputs?.tools) as StructuredTool[])?.map((tool) => [tool.name, tool]) ?? [])
const variables = await getVars(appDataSource, databaseEntities, nodeData)
const variables = await getVars(appDataSource, databaseEntities, nodeData, options)
const flow = {
chatflowId: options.chatflowid,
sessionId: options.sessionId,
packages/components/nodes/utilities/IfElseFunction/IfElseFunction.ts
+1 -1
View File
@@ -85,7 +85,7 @@ class IfElseFunction_Utilities implements INode {
const appDataSource = options.appDataSource as DataSource
const databaseEntities = options.databaseEntities as IDatabaseEntity
const variables = await getVars(appDataSource, databaseEntities, nodeData)
const variables = await getVars(appDataSource, databaseEntities, nodeData, options)
const flow = {
chatflowId: options.chatflowid,
sessionId: options.sessionId,
packages/components/nodes/vectorstores/Chroma/Chroma.ts
-1
View File
@@ -213,7 +213,6 @@ class Chroma_VectorStores implements INode {
const chromaApiKey = getCredentialParam('chromaApiKey', credentialData, nodeData)
const chromaTenant = getCredentialParam('chromaTenant', credentialData, nodeData)
const chromaDatabase = getCredentialParam('chromaDatabase', credentialData, nodeData)
const chromaMetadataFilter = nodeData.inputs?.chromaMetadataFilter
const obj: {
packages/components/nodes/vectorstores/DocumentStoreVS/DocStoreVector.ts
+2 -1
View File
@@ -56,7 +56,8 @@ class DocStore_VectorStores implements INode {
return returnData
}
const stores = await appDataSource.getRepository(databaseEntities['DocumentStore']).find()
const searchOptions = options.searchOptions || {}
const stores = await appDataSource.getRepository(databaseEntities['DocumentStore']).findBy(searchOptions)
for (const store of stores) {
if (store.status === 'UPSERTED') {
const obj = {
packages/components/nodes/vectorstores/Vectara/Vectara.ts
+2 -1
View File
@@ -191,11 +191,12 @@ class Vectara_VectorStores implements INode {
} else {
files = [fileName]
}
const orgId = options.orgId
const chatflowid = options.chatflowid
for (const file of files) {
if (!file) continue
const fileData = await getFileFromStorage(file, chatflowid)
const fileData = await getFileFromStorage(file, orgId, chatflowid)
const blob = new Blob([fileData])
vectaraFiles.push({ blob: blob, fileName: getFileName(file) })
}
packages/components/nodes/vectorstores/Vectara/Vectara_Upload.ts
+197
View File
@@ -0,0 +1,197 @@
import { VectaraStore, VectaraLibArgs, VectaraFilter, VectaraContextConfig, VectaraFile } from '@langchain/community/vectorstores/vectara'
import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
import { getFileFromStorage } from '../../../src'
class VectaraUpload_VectorStores implements INode {
label: string
name: string
version: number
description: string
type: string
icon: string
category: string
badge: string
baseClasses: string[]
inputs: INodeParams[]
credential: INodeParams
outputs: INodeOutputsValue[]
constructor() {
this.label = 'Vectara Upload File'
this.name = 'vectaraUpload'
this.version = 1.0
this.type = 'Vectara'
this.icon = 'vectara.png'
this.category = 'Vector Stores'
this.description = 'Upload files to Vectara'
this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
this.badge = 'DEPRECATING'
this.credential = {
label: 'Connect Credential',
name: 'credential',
type: 'credential',
credentialNames: ['vectaraApi']
}
this.inputs = [
{
label: 'File',
name: 'file',
description:
'File to upload to Vectara. Supported file types: https://docs.vectara.com/docs/api-reference/indexing-apis/file-upload/file-upload-filetypes',
type: 'file'
},
{
label: 'Metadata Filter',
name: 'filter',
description:
'Filter to apply to Vectara metadata. Refer to the <a target="_blank" href="https://docs.flowiseai.com/vector-stores/vectara">documentation</a> on how to use Vectara filters with Flowise.',
type: 'string',
additionalParams: true,
optional: true
},
{
label: 'Sentences Before',
name: 'sentencesBefore',
description: 'Number of sentences to fetch before the matched sentence. Defaults to 2.',
type: 'number',
additionalParams: true,
optional: true
},
{
label: 'Sentences After',
name: 'sentencesAfter',
description: 'Number of sentences to fetch after the matched sentence. Defaults to 2.',
type: 'number',
additionalParams: true,
optional: true
},
{
label: 'Lambda',
name: 'lambda',
description:
'Improves retrieval accuracy by adjusting the balance (from 0 to 1) between neural search and keyword-based search factors.',
type: 'number',
additionalParams: true,
optional: true
},
{
label: 'Top K',
name: 'topK',
description: 'Number of top results to fetch. Defaults to 4',
placeholder: '4',
type: 'number',
additionalParams: true,
optional: true
}
]
this.outputs = [
{
label: 'Vectara Retriever',
name: 'retriever',
baseClasses: this.baseClasses
},
{
label: 'Vectara Vector Store',
name: 'vectorStore',
baseClasses: [this.type, ...getBaseClasses(VectaraStore)]
}
]
}
async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
const apiKey = getCredentialParam('apiKey', credentialData, nodeData)
const customerId = getCredentialParam('customerID', credentialData, nodeData)
const corpusId = getCredentialParam('corpusID', credentialData, nodeData).split(',')
const fileBase64 = nodeData.inputs?.file
const vectaraMetadataFilter = nodeData.inputs?.filter as string
const sentencesBefore = nodeData.inputs?.sentencesBefore as number
const sentencesAfter = nodeData.inputs?.sentencesAfter as number
const lambda = nodeData.inputs?.lambda as number
const output = nodeData.outputs?.output as string
const topK = nodeData.inputs?.topK as string
const k = topK ? parseInt(topK, 10) : 4
const vectaraArgs: VectaraLibArgs = {
apiKey: apiKey,
customerId: customerId,
corpusId: corpusId,
source: 'flowise'
}
const vectaraFilter: VectaraFilter = {}
if (vectaraMetadataFilter) vectaraFilter.filter = vectaraMetadataFilter
if (lambda) vectaraFilter.lambda = lambda
const vectaraContextConfig: VectaraContextConfig = {}
if (sentencesBefore) vectaraContextConfig.sentencesBefore = sentencesBefore
if (sentencesAfter) vectaraContextConfig.sentencesAfter = sentencesAfter
vectaraFilter.contextConfig = vectaraContextConfig
let files: string[] = []
const vectaraFiles: VectaraFile[] = []
if (fileBase64.startsWith('FILE-STORAGE::')) {
const fileName = fileBase64.replace('FILE-STORAGE::', '')
if (fileName.startsWith('[') && fileName.endsWith(']')) {
files = JSON.parse(fileName)
} else {
files = [fileName]
}
const orgId = options.orgId
const chatflowid = options.chatflowid
for (const file of files) {
const fileData = await getFileFromStorage(file, orgId, chatflowid)
const blob = new Blob([fileData])
vectaraFiles.push({ blob: blob, fileName: getFileName(file) })
}
} else {
if (fileBase64.startsWith('[') && fileBase64.endsWith(']')) {
files = JSON.parse(fileBase64)
} else {
files = [fileBase64]
}
for (const file of files) {
const splitDataURI = file.split(',')
splitDataURI.pop()
const bf = Buffer.from(splitDataURI.pop() || '', 'base64')
const blob = new Blob([bf])
vectaraFiles.push({ blob: blob, fileName: getFileName(file) })
}
}
const vectorStore = new VectaraStore(vectaraArgs)
await vectorStore.addFiles(vectaraFiles)
if (output === 'retriever') {
const retriever = vectorStore.asRetriever(k, vectaraFilter)
return retriever
} else if (output === 'vectorStore') {
;(vectorStore as any).k = k
return vectorStore
}
return vectorStore
}
}
const getFileName = (fileBase64: string) => {
let fileNames = []
if (fileBase64.startsWith('[') && fileBase64.endsWith(']')) {
const files = JSON.parse(fileBase64)
for (const file of files) {
const splitDataURI = file.split(',')
const filename = splitDataURI[splitDataURI.length - 1].split(':')[1]
fileNames.push(filename)
}
return fileNames.join(', ')
} else {
const splitDataURI = fileBase64.split(',')
const filename = splitDataURI[splitDataURI.length - 1].split(':')[1]
return filename
}
}
module.exports = { nodeClass: VectaraUpload_VectorStores }