mirror of
https://github.com/farcasclaudiu/Flowise.git
synced 2026-06-28 13:00:56 +03:00
Feature/Add bullmq redis for message queue processing (#3568)
* add bullmq redis for message queue processing * Update pnpm-lock.yaml * update queue manager * remove singleton patterns, add redis to cache pool * add bull board ui * update rate limit handler * update redis configuration * Merge add rate limit redis prefix * update rate limit queue events * update preview loader to queue * refractor namings to constants * update env variable for queue * update worker shutdown gracefully
This commit is contained in:
Henry Heng
@@ -23,7 +23,7 @@ import {
|
||||
getAPIOverrideConfig
|
||||
} from '../utils'
|
||||
import { validateChatflowAPIKey } from './validateKey'
|
||||
import { IncomingInput, INodeDirectedGraph, IReactFlowObject, ChatType } from '../Interface'
|
||||
import { IncomingInput, INodeDirectedGraph, IReactFlowObject, ChatType, IExecuteFlowParams, MODE } from '../Interface'
|
||||
import { ChatFlow } from '../database/entities/ChatFlow'
|
||||
import { getRunningExpressApp } from '../utils/getRunningExpressApp'
|
||||
import { UpsertHistory } from '../database/entities/UpsertHistory'
|
||||
@@ -33,17 +33,182 @@ import { getErrorMessage } from '../errors/utils'
|
||||
import { v4 as uuidv4 } from 'uuid'
|
||||
import { FLOWISE_COUNTER_STATUS, FLOWISE_METRIC_COUNTERS } from '../Interface.Metrics'
|
||||
import { Variable } from '../database/entities/Variable'
|
||||
import { OMIT_QUEUE_JOB_DATA } from './constants'
|
||||
|
||||
export const executeUpsert = async ({
|
||||
componentNodes,
|
||||
incomingInput,
|
||||
chatflow,
|
||||
chatId,
|
||||
appDataSource,
|
||||
telemetry,
|
||||
cachePool,
|
||||
isInternal,
|
||||
files
|
||||
}: IExecuteFlowParams) => {
|
||||
const question = incomingInput.question
|
||||
const overrideConfig = incomingInput.overrideConfig ?? {}
|
||||
let stopNodeId = incomingInput?.stopNodeId ?? ''
|
||||
const chatHistory: IMessage[] = []
|
||||
const isUpsert = true
|
||||
const chatflowid = chatflow.id
|
||||
const apiMessageId = uuidv4()
|
||||
|
||||
if (files?.length) {
|
||||
const overrideConfig: ICommonObject = { ...incomingInput }
|
||||
for (const file of files) {
|
||||
const fileNames: string[] = []
|
||||
const fileBuffer = await getFileFromUpload(file.path ?? file.key)
|
||||
// Address file name with special characters: https://github.com/expressjs/multer/issues/1104
|
||||
file.originalname = Buffer.from(file.originalname, 'latin1').toString('utf8')
|
||||
const storagePath = await addArrayFilesToStorage(file.mimetype, fileBuffer, file.originalname, fileNames, chatflowid)
|
||||
|
||||
const fileInputFieldFromMimeType = mapMimeTypeToInputField(file.mimetype)
|
||||
|
||||
const fileExtension = path.extname(file.originalname)
|
||||
|
||||
const fileInputFieldFromExt = mapExtToInputField(fileExtension)
|
||||
|
||||
let fileInputField = 'txtFile'
|
||||
|
||||
if (fileInputFieldFromExt !== 'txtFile') {
|
||||
fileInputField = fileInputFieldFromExt
|
||||
} else if (fileInputFieldFromMimeType !== 'txtFile') {
|
||||
fileInputField = fileInputFieldFromExt
|
||||
}
|
||||
|
||||
if (overrideConfig[fileInputField]) {
|
||||
const existingFileInputField = overrideConfig[fileInputField].replace('FILE-STORAGE::', '')
|
||||
const existingFileInputFieldArray = JSON.parse(existingFileInputField)
|
||||
|
||||
const newFileInputField = storagePath.replace('FILE-STORAGE::', '')
|
||||
const newFileInputFieldArray = JSON.parse(newFileInputField)
|
||||
|
||||
const updatedFieldArray = existingFileInputFieldArray.concat(newFileInputFieldArray)
|
||||
|
||||
overrideConfig[fileInputField] = `FILE-STORAGE::${JSON.stringify(updatedFieldArray)}`
|
||||
} else {
|
||||
overrideConfig[fileInputField] = storagePath
|
||||
}
|
||||
|
||||
await removeSpecificFileFromUpload(file.path ?? file.key)
|
||||
}
|
||||
if (overrideConfig.vars && typeof overrideConfig.vars === 'string') {
|
||||
overrideConfig.vars = JSON.parse(overrideConfig.vars)
|
||||
}
|
||||
incomingInput = {
|
||||
...incomingInput,
|
||||
question: '',
|
||||
overrideConfig,
|
||||
stopNodeId,
|
||||
chatId
|
||||
}
|
||||
}
|
||||
|
||||
/*** Get chatflows and prepare data ***/
|
||||
const flowData = chatflow.flowData
|
||||
const parsedFlowData: IReactFlowObject = JSON.parse(flowData)
|
||||
const nodes = parsedFlowData.nodes
|
||||
const edges = parsedFlowData.edges
|
||||
|
||||
/*** Get session ID ***/
|
||||
const memoryNode = findMemoryNode(nodes, edges)
|
||||
let sessionId = getMemorySessionId(memoryNode, incomingInput, chatId, isInternal)
|
||||
|
||||
/*** Find the 1 final vector store will be upserted ***/
|
||||
const vsNodes = nodes.filter((node) => node.data.category === 'Vector Stores')
|
||||
const vsNodesWithFileUpload = vsNodes.filter((node) => node.data.inputs?.fileUpload)
|
||||
if (vsNodesWithFileUpload.length > 1) {
|
||||
throw new InternalFlowiseError(StatusCodes.INTERNAL_SERVER_ERROR, 'Multiple vector store nodes with fileUpload enabled')
|
||||
} else if (vsNodesWithFileUpload.length === 1 && !stopNodeId) {
|
||||
stopNodeId = vsNodesWithFileUpload[0].data.id
|
||||
}
|
||||
|
||||
/*** Check if multiple vector store nodes exist, and if stopNodeId is specified ***/
|
||||
if (vsNodes.length > 1 && !stopNodeId) {
|
||||
throw new InternalFlowiseError(
|
||||
StatusCodes.INTERNAL_SERVER_ERROR,
|
||||
'There are multiple vector nodes, please provide stopNodeId in body request'
|
||||
)
|
||||
} else if (vsNodes.length === 1 && !stopNodeId) {
|
||||
stopNodeId = vsNodes[0].data.id
|
||||
} else if (!vsNodes.length && !stopNodeId) {
|
||||
throw new InternalFlowiseError(StatusCodes.NOT_FOUND, 'No vector node found')
|
||||
}
|
||||
|
||||
/*** Get Starting Nodes with Reversed Graph ***/
|
||||
const { graph } = constructGraphs(nodes, edges, { isReversed: true })
|
||||
const nodeIds = getAllConnectedNodes(graph, stopNodeId)
|
||||
const filteredGraph: INodeDirectedGraph = {}
|
||||
for (const key of nodeIds) {
|
||||
if (Object.prototype.hasOwnProperty.call(graph, key)) {
|
||||
filteredGraph[key] = graph[key]
|
||||
}
|
||||
}
|
||||
const { startingNodeIds, depthQueue } = getStartingNodes(filteredGraph, stopNodeId)
|
||||
|
||||
/*** Get API Config ***/
|
||||
const availableVariables = await appDataSource.getRepository(Variable).find()
|
||||
const { nodeOverrides, variableOverrides, apiOverrideStatus } = getAPIOverrideConfig(chatflow)
|
||||
|
||||
const upsertedResult = await buildFlow({
|
||||
startingNodeIds,
|
||||
reactFlowNodes: nodes,
|
||||
reactFlowEdges: edges,
|
||||
apiMessageId,
|
||||
graph: filteredGraph,
|
||||
depthQueue,
|
||||
componentNodes,
|
||||
question,
|
||||
chatHistory,
|
||||
chatId,
|
||||
sessionId,
|
||||
chatflowid,
|
||||
appDataSource,
|
||||
overrideConfig,
|
||||
apiOverrideStatus,
|
||||
nodeOverrides,
|
||||
availableVariables,
|
||||
variableOverrides,
|
||||
cachePool,
|
||||
isUpsert,
|
||||
stopNodeId
|
||||
})
|
||||
|
||||
// Save to DB
|
||||
if (upsertedResult['flowData'] && upsertedResult['result']) {
|
||||
const result = cloneDeep(upsertedResult)
|
||||
result['flowData'] = JSON.stringify(result['flowData'])
|
||||
result['result'] = JSON.stringify(omit(result['result'], ['totalKeys', 'addedDocs']))
|
||||
result.chatflowid = chatflowid
|
||||
const newUpsertHistory = new UpsertHistory()
|
||||
Object.assign(newUpsertHistory, result)
|
||||
const upsertHistory = appDataSource.getRepository(UpsertHistory).create(newUpsertHistory)
|
||||
await appDataSource.getRepository(UpsertHistory).save(upsertHistory)
|
||||
}
|
||||
|
||||
await telemetry.sendTelemetry('vector_upserted', {
|
||||
version: await getAppVersion(),
|
||||
chatlowId: chatflowid,
|
||||
type: isInternal ? ChatType.INTERNAL : ChatType.EXTERNAL,
|
||||
flowGraph: getTelemetryFlowObj(nodes, edges),
|
||||
stopNodeId
|
||||
})
|
||||
|
||||
return upsertedResult['result'] ?? { result: 'Successfully Upserted' }
|
||||
}
|
||||
|
||||
/**
|
||||
* Upsert documents
|
||||
* @param {Request} req
|
||||
* @param {boolean} isInternal
|
||||
*/
|
||||
export const upsertVector = async (req: Request, isInternal: boolean = false) => {
|
||||
const appServer = getRunningExpressApp()
|
||||
try {
|
||||
const appServer = getRunningExpressApp()
|
||||
const chatflowid = req.params.id
|
||||
let incomingInput: IncomingInput = req.body
|
||||
|
||||
// Check if chatflow exists
|
||||
const chatflow = await appServer.AppDataSource.getRepository(ChatFlow).findOneBy({
|
||||
id: chatflowid
|
||||
})
|
||||
@@ -51,6 +216,12 @@ export const upsertVector = async (req: Request, isInternal: boolean = false) =>
|
||||
throw new InternalFlowiseError(StatusCodes.NOT_FOUND, `Chatflow ${chatflowid} not found`)
|
||||
}
|
||||
|
||||
const httpProtocol = req.get('x-forwarded-proto') || req.protocol
|
||||
const baseURL = `${httpProtocol}://${req.get('host')}`
|
||||
const incomingInput: IncomingInput = req.body
|
||||
const chatId = incomingInput.chatId ?? incomingInput.overrideConfig?.sessionId ?? uuidv4()
|
||||
const files = (req.files as Express.Multer.File[]) || []
|
||||
|
||||
if (!isInternal) {
|
||||
const isKeyValidated = await validateChatflowAPIKey(req, chatflow)
|
||||
if (!isKeyValidated) {
|
||||
@@ -58,168 +229,50 @@ export const upsertVector = async (req: Request, isInternal: boolean = false) =>
|
||||
}
|
||||
}
|
||||
|
||||
const files = (req.files as Express.Multer.File[]) || []
|
||||
|
||||
if (files.length) {
|
||||
const overrideConfig: ICommonObject = { ...req.body }
|
||||
for (const file of files) {
|
||||
const fileNames: string[] = []
|
||||
const fileBuffer = await getFileFromUpload(file.path ?? file.key)
|
||||
// Address file name with special characters: https://github.com/expressjs/multer/issues/1104
|
||||
file.originalname = Buffer.from(file.originalname, 'latin1').toString('utf8')
|
||||
const storagePath = await addArrayFilesToStorage(file.mimetype, fileBuffer, file.originalname, fileNames, chatflowid)
|
||||
|
||||
const fileInputFieldFromMimeType = mapMimeTypeToInputField(file.mimetype)
|
||||
|
||||
const fileExtension = path.extname(file.originalname)
|
||||
|
||||
const fileInputFieldFromExt = mapExtToInputField(fileExtension)
|
||||
|
||||
let fileInputField = 'txtFile'
|
||||
|
||||
if (fileInputFieldFromExt !== 'txtFile') {
|
||||
fileInputField = fileInputFieldFromExt
|
||||
} else if (fileInputFieldFromMimeType !== 'txtFile') {
|
||||
fileInputField = fileInputFieldFromExt
|
||||
}
|
||||
|
||||
if (overrideConfig[fileInputField]) {
|
||||
const existingFileInputField = overrideConfig[fileInputField].replace('FILE-STORAGE::', '')
|
||||
const existingFileInputFieldArray = JSON.parse(existingFileInputField)
|
||||
|
||||
const newFileInputField = storagePath.replace('FILE-STORAGE::', '')
|
||||
const newFileInputFieldArray = JSON.parse(newFileInputField)
|
||||
|
||||
const updatedFieldArray = existingFileInputFieldArray.concat(newFileInputFieldArray)
|
||||
|
||||
overrideConfig[fileInputField] = `FILE-STORAGE::${JSON.stringify(updatedFieldArray)}`
|
||||
} else {
|
||||
overrideConfig[fileInputField] = storagePath
|
||||
}
|
||||
|
||||
await removeSpecificFileFromUpload(file.path ?? file.key)
|
||||
}
|
||||
if (overrideConfig.vars && typeof overrideConfig.vars === 'string') {
|
||||
overrideConfig.vars = JSON.parse(overrideConfig.vars)
|
||||
}
|
||||
incomingInput = {
|
||||
question: req.body.question ?? 'hello',
|
||||
overrideConfig,
|
||||
stopNodeId: req.body.stopNodeId
|
||||
}
|
||||
if (req.body.chatId) {
|
||||
incomingInput.chatId = req.body.chatId
|
||||
}
|
||||
}
|
||||
|
||||
/*** Get chatflows and prepare data ***/
|
||||
const flowData = chatflow.flowData
|
||||
const parsedFlowData: IReactFlowObject = JSON.parse(flowData)
|
||||
const nodes = parsedFlowData.nodes
|
||||
const edges = parsedFlowData.edges
|
||||
|
||||
const apiMessageId = req.body.apiMessageId ?? uuidv4()
|
||||
|
||||
let stopNodeId = incomingInput?.stopNodeId ?? ''
|
||||
let chatHistory: IMessage[] = []
|
||||
let chatId = incomingInput.chatId ?? ''
|
||||
let isUpsert = true
|
||||
|
||||
// Get session ID
|
||||
const memoryNode = findMemoryNode(nodes, edges)
|
||||
let sessionId = getMemorySessionId(memoryNode, incomingInput, chatId, isInternal)
|
||||
|
||||
const vsNodes = nodes.filter((node) => node.data.category === 'Vector Stores')
|
||||
|
||||
// Get StopNodeId for vector store which has fielUpload
|
||||
const vsNodesWithFileUpload = vsNodes.filter((node) => node.data.inputs?.fileUpload)
|
||||
if (vsNodesWithFileUpload.length > 1) {
|
||||
throw new InternalFlowiseError(StatusCodes.INTERNAL_SERVER_ERROR, 'Multiple vector store nodes with fileUpload enabled')
|
||||
} else if (vsNodesWithFileUpload.length === 1 && !stopNodeId) {
|
||||
stopNodeId = vsNodesWithFileUpload[0].data.id
|
||||
}
|
||||
|
||||
// Check if multiple vector store nodes exist, and if stopNodeId is specified
|
||||
if (vsNodes.length > 1 && !stopNodeId) {
|
||||
throw new InternalFlowiseError(
|
||||
StatusCodes.INTERNAL_SERVER_ERROR,
|
||||
'There are multiple vector nodes, please provide stopNodeId in body request'
|
||||
)
|
||||
} else if (vsNodes.length === 1 && !stopNodeId) {
|
||||
stopNodeId = vsNodes[0].data.id
|
||||
} else if (!vsNodes.length && !stopNodeId) {
|
||||
throw new InternalFlowiseError(StatusCodes.NOT_FOUND, 'No vector node found')
|
||||
}
|
||||
|
||||
const { graph } = constructGraphs(nodes, edges, { isReversed: true })
|
||||
|
||||
const nodeIds = getAllConnectedNodes(graph, stopNodeId)
|
||||
|
||||
const filteredGraph: INodeDirectedGraph = {}
|
||||
for (const key of nodeIds) {
|
||||
if (Object.prototype.hasOwnProperty.call(graph, key)) {
|
||||
filteredGraph[key] = graph[key]
|
||||
}
|
||||
}
|
||||
|
||||
const { startingNodeIds, depthQueue } = getStartingNodes(filteredGraph, stopNodeId)
|
||||
|
||||
/*** Get API Config ***/
|
||||
const availableVariables = await appServer.AppDataSource.getRepository(Variable).find()
|
||||
const { nodeOverrides, variableOverrides, apiOverrideStatus } = getAPIOverrideConfig(chatflow)
|
||||
|
||||
const upsertedResult = await buildFlow({
|
||||
startingNodeIds,
|
||||
reactFlowNodes: nodes,
|
||||
reactFlowEdges: edges,
|
||||
apiMessageId,
|
||||
graph: filteredGraph,
|
||||
depthQueue,
|
||||
const executeData: IExecuteFlowParams = {
|
||||
componentNodes: appServer.nodesPool.componentNodes,
|
||||
question: incomingInput.question,
|
||||
chatHistory,
|
||||
incomingInput,
|
||||
chatflow,
|
||||
chatId,
|
||||
sessionId: sessionId ?? '',
|
||||
chatflowid,
|
||||
appDataSource: appServer.AppDataSource,
|
||||
overrideConfig: incomingInput?.overrideConfig,
|
||||
apiOverrideStatus,
|
||||
nodeOverrides,
|
||||
availableVariables,
|
||||
variableOverrides,
|
||||
telemetry: appServer.telemetry,
|
||||
cachePool: appServer.cachePool,
|
||||
isUpsert,
|
||||
stopNodeId
|
||||
})
|
||||
|
||||
const startingNodes = nodes.filter((nd) => startingNodeIds.includes(nd.data.id))
|
||||
|
||||
await appServer.chatflowPool.add(chatflowid, undefined, startingNodes, incomingInput?.overrideConfig, chatId)
|
||||
|
||||
// Save to DB
|
||||
if (upsertedResult['flowData'] && upsertedResult['result']) {
|
||||
const result = cloneDeep(upsertedResult)
|
||||
result['flowData'] = JSON.stringify(result['flowData'])
|
||||
result['result'] = JSON.stringify(omit(result['result'], ['totalKeys', 'addedDocs']))
|
||||
result.chatflowid = chatflowid
|
||||
const newUpsertHistory = new UpsertHistory()
|
||||
Object.assign(newUpsertHistory, result)
|
||||
const upsertHistory = appServer.AppDataSource.getRepository(UpsertHistory).create(newUpsertHistory)
|
||||
await appServer.AppDataSource.getRepository(UpsertHistory).save(upsertHistory)
|
||||
sseStreamer: appServer.sseStreamer,
|
||||
baseURL,
|
||||
isInternal,
|
||||
files,
|
||||
isUpsert: true
|
||||
}
|
||||
|
||||
await appServer.telemetry.sendTelemetry('vector_upserted', {
|
||||
version: await getAppVersion(),
|
||||
chatlowId: chatflowid,
|
||||
type: isInternal ? ChatType.INTERNAL : ChatType.EXTERNAL,
|
||||
flowGraph: getTelemetryFlowObj(nodes, edges),
|
||||
stopNodeId
|
||||
})
|
||||
appServer.metricsProvider?.incrementCounter(FLOWISE_METRIC_COUNTERS.VECTORSTORE_UPSERT, { status: FLOWISE_COUNTER_STATUS.SUCCESS })
|
||||
if (process.env.MODE === MODE.QUEUE) {
|
||||
const upsertQueue = appServer.queueManager.getQueue('upsert')
|
||||
|
||||
return upsertedResult['result'] ?? { result: 'Successfully Upserted' }
|
||||
const job = await upsertQueue.addJob(omit(executeData, OMIT_QUEUE_JOB_DATA))
|
||||
logger.debug(`[server]: Job added to queue: ${job.id}`)
|
||||
|
||||
const queueEvents = upsertQueue.getQueueEvents()
|
||||
const result = await job.waitUntilFinished(queueEvents)
|
||||
|
||||
if (!result) {
|
||||
throw new Error('Job execution failed')
|
||||
}
|
||||
|
||||
appServer.metricsProvider?.incrementCounter(FLOWISE_METRIC_COUNTERS.VECTORSTORE_UPSERT, {
|
||||
status: FLOWISE_COUNTER_STATUS.SUCCESS
|
||||
})
|
||||
return result
|
||||
} else {
|
||||
const result = await executeUpsert(executeData)
|
||||
|
||||
appServer.metricsProvider?.incrementCounter(FLOWISE_METRIC_COUNTERS.VECTORSTORE_UPSERT, {
|
||||
status: FLOWISE_COUNTER_STATUS.SUCCESS
|
||||
})
|
||||
return result
|
||||
}
|
||||
} catch (e) {
|
||||
logger.error('[server]: Error:', e)
|
||||
appServer.metricsProvider?.incrementCounter(FLOWISE_METRIC_COUNTERS.VECTORSTORE_UPSERT, { status: FLOWISE_COUNTER_STATUS.FAILURE })
|
||||
|
||||
if (e instanceof InternalFlowiseError && e.statusCode === StatusCodes.UNAUTHORIZED) {
|
||||
throw e
|
||||
} else {
|
||||
|
||||
Reference in New Issue
Block a user