farcasclaudiu/Flowise
1
0
Fork 0
mirror of https://github.com/farcasclaudiu/Flowise.git synced 2026-06-28 13:00:56 +03:00
Code Issues Packages Projects Releases Wiki Activity

Chore/refractor (#4454)

Browse Source 
* markdown files and env examples cleanup

* components update

* update jsonlines description

* server refractor

* update telemetry

* add execute custom node

* add ui refractor

* add username and password authenticate

* correctly retrieve past images in agentflowv2

* disable e2e temporarily

* add existing username and password authenticate

* update migration to default workspace

* update todo

* blob storage migrating

* throw error on agent tool call error

* add missing execution import

* add referral

* chore: add error message when importData is undefined

* migrate api keys to db

* fix: data too long for column executionData

* migrate api keys from json to db at init

* add info on account setup

* update docstore missing fields

---------

Co-authored-by: chungyau97 <chungyau97@gmail.com>
This commit is contained in:
Henry Heng
2025-05-27 14:29:42 +08:00
committed by GitHub
parent e35a126b46
commit 5a37227d14
560 changed files with 62127 additions and 4100 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/server/src/utils/addChatflowsCount.ts
+1
View File
@@ -15,6 +15,7 @@ export const addChatflowsCount = async (keys: any) => {
const chatflows = await appServer.AppDataSource.getRepository(ChatFlow)
.createQueryBuilder('cf')
.where('cf.apikeyid = :apikeyid', { apikeyid: key.id })
.andWhere('cf.workspaceId = :workspaceId', { workspaceId: key.workspaceId })
.getMany()
const linkedChatFlows: any[] = []
chatflows.map((cf) => {
packages/server/src/utils/apiKey.ts
+83 -119
View File
@@ -1,10 +1,14 @@
import { randomBytes, scryptSync, timingSafeEqual } from 'crypto'
import { ICommonObject } from 'flowise-components'
import moment from 'moment'
import fs from 'fs'
import path from 'path'
import logger from './logger'
import { appConfig } from '../AppConfig'
import { DataSource } from 'typeorm'
import { ApiKey } from '../database/entities/ApiKey'
import { Workspace } from '../enterprise/database/entities/workspace.entity'
import { v4 as uuidv4 } from 'uuid'
import { ChatFlow } from '../database/entities/ChatFlow'
import { addChatflowsCount } from './addChatflowsCount'
import { Platform } from '../Interface'
/**
* Returns the api key path
@@ -51,94 +55,14 @@ export const compareKeys = (storedKey: string, suppliedKey: string): boolean =>
* @returns {Promise<ICommonObject[]>}
*/
export const getAPIKeys = async (): Promise<ICommonObject[]> => {
if (appConfig.apiKeys.storageType !== 'json') {
return []
}
try {
const content = await fs.promises.readFile(getAPIKeyPath(), 'utf8')
return JSON.parse(content)
} catch (error) {
const keyName = 'DefaultKey'
const apiKey = generateAPIKey()
const apiSecret = generateSecretHash(apiKey)
const content = [
{
keyName,
apiKey,
apiSecret,
createdAt: moment().format('DD-MMM-YY'),
id: randomBytes(16).toString('hex')
}
]
await fs.promises.writeFile(getAPIKeyPath(), JSON.stringify(content), 'utf8')
return content
return []
}
}
/**
* Add new API key
* @param {string} keyName
* @returns {Promise<ICommonObject[]>}
*/
export const addAPIKey = async (keyName: string): Promise<ICommonObject[]> => {
const existingAPIKeys = await getAPIKeys()
const apiKey = generateAPIKey()
const apiSecret = generateSecretHash(apiKey)
const content = [
...existingAPIKeys,
{
keyName,
apiKey,
apiSecret,
createdAt: moment().format('DD-MMM-YY'),
id: randomBytes(16).toString('hex')
}
]
await fs.promises.writeFile(getAPIKeyPath(), JSON.stringify(content), 'utf8')
return content
}
/**
* import API keys
* @param {[]} keys
* @returns {Promise<ICommonObject[]>}
*/
export const importKeys = async (keys: any[], importMode: string): Promise<ICommonObject[]> => {
const allApiKeys = await getAPIKeys()
// if importMode is errorIfExist, check for existing keys and raise error before any modification to the file
if (importMode === 'errorIfExist') {
for (const key of keys) {
const keyNameExists = allApiKeys.find((k) => k.keyName === key.keyName)
if (keyNameExists) {
throw new Error(`Key with name ${key.keyName} already exists`)
}
}
}
for (const key of keys) {
// Check if keyName already exists, if overwrite is false, raise an error else overwrite the key
const keyNameExists = allApiKeys.find((k) => k.keyName === key.keyName)
if (keyNameExists) {
const keyIndex = allApiKeys.findIndex((k) => k.keyName === key.keyName)
switch (importMode) {
case 'overwriteIfExist':
allApiKeys[keyIndex] = key
continue
case 'ignoreIfExist':
// ignore this key and continue
continue
case 'errorIfExist':
// should not reach here as we have already checked for existing keys
throw new Error(`Key with name ${key.keyName} already exists`)
default:
throw new Error(`Unknown overwrite option ${importMode}`)
}
}
allApiKeys.push(key)
}
await fs.promises.writeFile(getAPIKeyPath(), JSON.stringify(allApiKeys), 'utf8')
return allApiKeys
}
/**
* Get API Key details
* @param {string} apiKey
@@ -151,42 +75,82 @@ export const getApiKey = async (apiKey: string) => {
return existingAPIKeys[keyIndex]
}
/**
* Update existing API key
* @param {string} keyIdToUpdate
* @param {string} newKeyName
* @returns {Promise<ICommonObject[]>}
*/
export const updateAPIKey = async (keyIdToUpdate: string, newKeyName: string): Promise<ICommonObject[]> => {
const existingAPIKeys = await getAPIKeys()
const keyIndex = existingAPIKeys.findIndex((key) => key.id === keyIdToUpdate)
if (keyIndex < 0) return []
existingAPIKeys[keyIndex].keyName = newKeyName
await fs.promises.writeFile(getAPIKeyPath(), JSON.stringify(existingAPIKeys), 'utf8')
return existingAPIKeys
}
export const migrateApiKeysFromJsonToDb = async (appDataSource: DataSource, platformType: Platform) => {
if (platformType === Platform.CLOUD) {
return
}
/**
* Delete API key
* @param {string} keyIdToDelete
* @returns {Promise<ICommonObject[]>}
*/
export const deleteAPIKey = async (keyIdToDelete: string): Promise<ICommonObject[]> => {
const existingAPIKeys = await getAPIKeys()
const result = existingAPIKeys.filter((key) => key.id !== keyIdToDelete)
await fs.promises.writeFile(getAPIKeyPath(), JSON.stringify(result), 'utf8')
return result
}
if (!process.env.APIKEY_STORAGE_TYPE || process.env.APIKEY_STORAGE_TYPE === 'json') {
const keys = await getAPIKeys()
if (keys.length > 0) {
try {
// Get all available workspaces
const workspaces = await appDataSource.getRepository(Workspace).find()
/**
* Replace all api keys
* @param {ICommonObject[]} content
* @returns {Promise<void>}
*/
export const replaceAllAPIKeys = async (content: ICommonObject[]): Promise<void> => {
try {
await fs.promises.writeFile(getAPIKeyPath(), JSON.stringify(content), 'utf8')
} catch (error) {
logger.error(error)
for (const key of keys) {
const existingKey = await appDataSource.getRepository(ApiKey).findOneBy({
apiKey: key.apiKey
})
// Only add if key doesn't already exist in DB
if (!existingKey) {
// Create a new API key for each workspace
if (workspaces.length > 0) {
for (const workspace of workspaces) {
const newKey = new ApiKey()
newKey.id = uuidv4()
newKey.apiKey = key.apiKey
newKey.apiSecret = key.apiSecret
newKey.keyName = key.keyName
newKey.workspaceId = workspace.id
const keyEntity = appDataSource.getRepository(ApiKey).create(newKey)
await appDataSource.getRepository(ApiKey).save(keyEntity)
const chatflows = await appDataSource.getRepository(ChatFlow).findBy({
apikeyid: key.id,
workspaceId: workspace.id
})
for (const chatflow of chatflows) {
chatflow.apikeyid = newKey.id
await appDataSource.getRepository(ChatFlow).save(chatflow)
}
await addChatflowsCount(chatflows)
}
} else {
// If no workspaces exist, create the key without a workspace ID and later will be updated by setNullWorkspaceId
const newKey = new ApiKey()
newKey.id = uuidv4()
newKey.apiKey = key.apiKey
newKey.apiSecret = key.apiSecret
newKey.keyName = key.keyName
const keyEntity = appDataSource.getRepository(ApiKey).create(newKey)
await appDataSource.getRepository(ApiKey).save(keyEntity)
const chatflows = await appDataSource.getRepository(ChatFlow).findBy({
apikeyid: key.id
})
for (const chatflow of chatflows) {
chatflow.apikeyid = newKey.id
await appDataSource.getRepository(ChatFlow).save(chatflow)
}
await addChatflowsCount(chatflows)
}
}
}
// Delete the JSON file
if (fs.existsSync(getAPIKeyPath())) {
fs.unlinkSync(getAPIKeyPath())
}
} catch (error) {
console.error('Error migrating API keys from JSON to DB', error)
}
}
}
}
packages/server/src/utils/buildAgentGraph.ts
+15 -8
View File
@@ -26,6 +26,7 @@ import { InternalFlowiseError } from '../errors/internalFlowiseError'
import { getErrorMessage } from '../errors/utils'
import logger from './logger'
import { Variable } from '../database/entities/Variable'
import { getWorkspaceSearchOptions } from '../enterprise/utils/ControllerServiceUtils'
import { DataSource } from 'typeorm'
import { CachePool } from '../CachePool'
@@ -50,7 +51,9 @@ export const buildAgentGraph = async ({
shouldStreamResponse,
cachePool,
baseURL,
signal
signal,
orgId,
workspaceId
}: {
agentflow: IChatFlow
flowConfig: IFlowConfig
@@ -70,6 +73,8 @@ export const buildAgentGraph = async ({
cachePool: CachePool
baseURL: string
signal?: AbortController
orgId: string
workspaceId?: string
}): Promise<any> => {
try {
const chatflowid = flowConfig.chatflowid
@@ -79,6 +84,8 @@ export const buildAgentGraph = async ({
const uploads = incomingInput.uploads
const options = {
orgId,
workspaceId,
chatId,
sessionId,
chatflowid,
@@ -384,7 +391,7 @@ export const buildAgentGraph = async ({
}
} catch (e) {
// clear agent memory because checkpoints were saved during runtime
await clearSessionMemory(nodes, componentNodes, chatId, appDataSource, sessionId)
await clearSessionMemory(nodes, componentNodes, chatId, appDataSource, orgId, sessionId)
if (getErrorMessage(e).includes('Aborted')) {
if (shouldStreamResponse && sseStreamer) {
sseStreamer.streamAbortEvent(chatId)
@@ -395,7 +402,7 @@ export const buildAgentGraph = async ({
}
return streamResults
} catch (e) {
logger.error('[server]: Error:', e)
logger.error(`[server]: [${orgId}]: Error:`, e)
throw new InternalFlowiseError(StatusCodes.INTERNAL_SERVER_ERROR, `Error buildAgentGraph - ${getErrorMessage(e)}`)
}
}
@@ -457,7 +464,7 @@ const compileMultiAgentsGraph = async (params: MultiAgentsGraphParams) => {
const workerNodes = reactFlowNodes.filter((node) => workerNodeIds.includes(node.data.id))
/*** Get API Config ***/
const availableVariables = await appDataSource.getRepository(Variable).find()
const availableVariables = await appDataSource.getRepository(Variable).findBy(getWorkspaceSearchOptions(agentflow.workspaceId))
const { nodeOverrides, variableOverrides, apiOverrideStatus } = getAPIOverrideConfig(agentflow)
let supervisorWorkers: { [key: string]: IMultiAgentNode[] } = {}
@@ -566,7 +573,7 @@ const compileMultiAgentsGraph = async (params: MultiAgentsGraphParams) => {
const graph = workflowGraph.compile({ checkpointer: memory })
const loggerHandler = new ConsoleCallbackHandler(logger)
const loggerHandler = new ConsoleCallbackHandler(logger, options?.orgId)
const callbacks = await additionalCallbacks(flowNodeData, options)
const config = { configurable: { thread_id: threadId } }
@@ -686,7 +693,7 @@ const compileSeqAgentsGraph = async (params: SeqAgentsGraphParams) => {
let interruptToolNodeNames = []
/*** Get API Config ***/
const availableVariables = await appDataSource.getRepository(Variable).find()
const availableVariables = await appDataSource.getRepository(Variable).findBy(getWorkspaceSearchOptions(agentflow.workspaceId))
const { nodeOverrides, variableOverrides, apiOverrideStatus } = getAPIOverrideConfig(agentflow)
const initiateNode = async (node: IReactFlowNode) => {
@@ -996,7 +1003,7 @@ const compileSeqAgentsGraph = async (params: SeqAgentsGraphParams) => {
interruptBefore: interruptToolNodeNames as any
})
const loggerHandler = new ConsoleCallbackHandler(logger)
const loggerHandler = new ConsoleCallbackHandler(logger, options?.orgId)
const callbacks = await additionalCallbacks(flowNodeData as any, options)
const config = { configurable: { thread_id: threadId }, bindModel }
@@ -1044,7 +1051,7 @@ const compileSeqAgentsGraph = async (params: SeqAgentsGraphParams) => {
configurable: config
})
} catch (e) {
logger.error('Error compile graph', e)
logger.error(`[${options.orgId}]: Error compile graph`, e)
throw new InternalFlowiseError(StatusCodes.INTERNAL_SERVER_ERROR, `Error compile graph - ${getErrorMessage(e)}`)
}
}
packages/server/src/utils/buildAgentflow.ts
+68 -24
View File
@@ -52,6 +52,8 @@ import { utilAddChatMessage } from './addChatMesage'
import { CachePool } from '../CachePool'
import { ChatMessage } from '../database/entities/ChatMessage'
import { Telemetry } from './telemetry'
import { getWorkspaceSearchOptions } from '../enterprise/utils/ControllerServiceUtils'
import { UsageCacheManager } from '../UsageCacheManager'
interface IWaitingNode {
nodeId: string
@@ -99,9 +101,11 @@ interface IExecuteNodeParams {
chatId: string
sessionId: string
apiMessageId: string
evaluationRunId?: string
isInternal: boolean
pastChatHistory: IMessage[]
appDataSource: DataSource
usageCacheManager: UsageCacheManager
telemetry: Telemetry
componentNodes: IComponentNodes
cachePool: CachePool
@@ -122,6 +126,9 @@ interface IExecuteNodeParams {
parentExecutionId?: string
isRecursive?: boolean
iterationContext?: ICommonObject
orgId: string
workspaceId: string
subscriptionId: string
}
interface IExecuteAgentFlowParams extends Omit<IExecuteFlowParams, 'incomingInput'> {
@@ -142,13 +149,15 @@ const addExecution = async (
appDataSource: DataSource,
agentflowId: string,
agentFlowExecutedData: IAgentflowExecutedData[],
sessionId: string
sessionId: string,
workspaceId: string
) => {
const newExecution = new Execution()
const bodyExecution = {
agentflowId,
state: 'INPROGRESS',
sessionId,
workspaceId,
executionData: JSON.stringify(agentFlowExecutedData)
}
Object.assign(newExecution, bodyExecution)
@@ -164,9 +173,10 @@ const addExecution = async (
* @param {Partial<IExecution>} data
* @returns {Promise<void>}
*/
const updateExecution = async (appDataSource: DataSource, executionId: string, data?: Partial<IExecution>) => {
const updateExecution = async (appDataSource: DataSource, executionId: string, workspaceId: string, data?: Partial<IExecution>) => {
const execution = await appDataSource.getRepository(Execution).findOneBy({
id: executionId
id: executionId,
workspaceId
})
if (!execution) {
@@ -770,9 +780,11 @@ const executeNode = async ({
chatId,
sessionId,
apiMessageId,
evaluationRunId,
parentExecutionId,
pastChatHistory,
appDataSource,
usageCacheManager,
telemetry,
componentNodes,
cachePool,
@@ -792,7 +804,10 @@ const executeNode = async ({
analyticHandlers,
isInternal,
isRecursive,
iterationContext
iterationContext,
orgId,
workspaceId,
subscriptionId
}: IExecuteNodeParams): Promise<{
result: any
shouldStop?: boolean
@@ -824,7 +839,7 @@ const executeNode = async ({
}
// Get available variables and resolve them
const availableVariables = await appDataSource.getRepository(Variable).find()
const availableVariables = await appDataSource.getRepository(Variable).findBy(getWorkspaceSearchOptions(workspaceId))
// Prepare flow config
let updatedState = cloneDeep(agentflowRuntime.state)
@@ -902,6 +917,9 @@ const executeNode = async ({
// Prepare run parameters
const runParams = {
orgId,
workspaceId,
subscriptionId,
chatId,
sessionId,
chatflowid: chatflow.id,
@@ -909,6 +927,7 @@ const executeNode = async ({
logger,
appDataSource,
databaseEntities,
usageCacheManager,
componentNodes,
cachePool,
analytic: chatflow.analytic,
@@ -922,7 +941,8 @@ const executeNode = async ({
analyticHandlers,
parentTraceIds,
humanInputAction,
iterationContext
iterationContext,
evaluationRunId
}
// Execute node
@@ -982,7 +1002,9 @@ const executeNode = async ({
incomingInput,
chatflow: iterationChatflow,
chatId,
evaluationRunId,
appDataSource,
usageCacheManager,
telemetry,
cachePool,
sseStreamer,
@@ -996,7 +1018,10 @@ const executeNode = async ({
iterationContext: {
...iterationContext,
agentflowRuntime
}
},
orgId,
workspaceId,
subscriptionId
})
// Store the result
@@ -1023,7 +1048,7 @@ const executeNode = async ({
if (parentExecutionId) {
try {
logger.debug(` 📝 Updating parent execution ${parentExecutionId} with iteration ${i + 1} data`)
await updateExecution(appDataSource, parentExecutionId, {
await updateExecution(appDataSource, parentExecutionId, workspaceId, {
executionData: JSON.stringify(agentFlowExecutedData)
})
} catch (error) {
@@ -1192,8 +1217,10 @@ export const executeAgentFlow = async ({
incomingInput,
chatflow,
chatId,
evaluationRunId,
appDataSource,
telemetry,
usageCacheManager,
cachePool,
sseStreamer,
baseURL,
@@ -1204,7 +1231,10 @@ export const executeAgentFlow = async ({
isRecursive = false,
parentExecutionId,
iterationContext,
isTool = false
isTool = false,
orgId,
workspaceId,
subscriptionId
}: IExecuteAgentFlowParams) => {
logger.debug('\n🚀 Starting flow execution')
@@ -1281,7 +1311,8 @@ export const executeAgentFlow = async ({
const previousExecutions = await appDataSource.getRepository(Execution).find({
where: {
sessionId,
agentflowId: chatflowid
agentflowId: chatflowid,
workspaceId
},
order: {
createdDate: 'DESC'
@@ -1344,7 +1375,7 @@ export const executeAgentFlow = async ({
agentflowRuntime.state = (lastState as ICommonObject) ?? {}
// Update execution state to INPROGRESS
await updateExecution(appDataSource, previousExecution.id, {
await updateExecution(appDataSource, previousExecution.id, workspaceId, {
state: 'INPROGRESS'
})
newExecution = previousExecution
@@ -1357,7 +1388,7 @@ export const executeAgentFlow = async ({
// For recursive calls with a valid parent execution ID, don't create a new execution
// Instead, fetch the parent execution to use it
const parentExecution = await appDataSource.getRepository(Execution).findOne({
where: { id: parentExecutionId }
where: { id: parentExecutionId, workspaceId }
})
if (parentExecution) {
@@ -1365,7 +1396,7 @@ export const executeAgentFlow = async ({
newExecution = parentExecution
} else {
console.warn(` ⚠️ Parent execution ID ${parentExecutionId} not found, will create new execution`)
newExecution = await addExecution(appDataSource, chatflowid, agentFlowExecutedData, sessionId)
newExecution = await addExecution(appDataSource, chatflowid, agentFlowExecutedData, sessionId, workspaceId)
parentExecutionId = newExecution.id
}
} else {
@@ -1374,7 +1405,7 @@ export const executeAgentFlow = async ({
checkForMultipleStartNodes(startingNodeIds, isRecursive, nodes)
// Only create a new execution if this is not a recursive call
newExecution = await addExecution(appDataSource, chatflowid, agentFlowExecutedData, sessionId)
newExecution = await addExecution(appDataSource, chatflowid, agentFlowExecutedData, sessionId, workspaceId)
parentExecutionId = newExecution.id
}
@@ -1430,6 +1461,8 @@ export const executeAgentFlow = async ({
try {
if (chatflow.analytic) {
analyticHandlers = AnalyticHandler.getInstance({ inputs: {} } as any, {
orgId,
workspaceId,
appDataSource,
databaseEntities,
componentNodes,
@@ -1486,10 +1519,12 @@ export const executeAgentFlow = async ({
chatId,
sessionId,
apiMessageId,
evaluationRunId,
parentExecutionId,
isInternal,
pastChatHistory,
appDataSource,
usageCacheManager,
telemetry,
componentNodes,
cachePool,
@@ -1508,7 +1543,10 @@ export const executeAgentFlow = async ({
parentTraceIds,
analyticHandlers,
isRecursive,
iterationContext
iterationContext,
orgId,
workspaceId,
subscriptionId
})
if (executionResult.agentFlowExecutedData) {
@@ -1607,7 +1645,7 @@ export const executeAgentFlow = async ({
if (!isRecursive) {
sseStreamer?.streamAgentFlowExecutedDataEvent(chatId, agentFlowExecutedData)
await updateExecution(appDataSource, newExecution.id, {
await updateExecution(appDataSource, newExecution.id, workspaceId, {
executionData: JSON.stringify(agentFlowExecutedData),
state: errorStatus
})
@@ -1642,7 +1680,7 @@ export const executeAgentFlow = async ({
// Only update execution record if this is not a recursive call
if (!isRecursive) {
await updateExecution(appDataSource, newExecution.id, {
await updateExecution(appDataSource, newExecution.id, workspaceId, {
executionData: JSON.stringify(agentFlowExecutedData),
state: status
})
@@ -1744,6 +1782,8 @@ export const executeAgentFlow = async ({
if (chatflow.followUpPrompts) {
const followUpPromptsConfig = JSON.parse(chatflow.followUpPrompts)
const followUpPrompts = await generateFollowUpPrompts(followUpPromptsConfig, apiMessage.content, {
orgId,
workspaceId,
chatId,
chatflowid,
appDataSource,
@@ -1760,13 +1800,17 @@ export const executeAgentFlow = async ({
logger.debug(`[server]: Finished running agentflow ${chatflowid}`)
await telemetry.sendTelemetry('prediction_sent', {
version: await getAppVersion(),
chatflowId: chatflowid,
chatId,
type: isInternal ? ChatType.INTERNAL : ChatType.EXTERNAL,
flowGraph: getTelemetryFlowObj(nodes, edges)
})
await telemetry.sendTelemetry(
'prediction_sent',
{
version: await getAppVersion(),
chatflowId: chatflowid,
chatId,
type: isInternal ? ChatType.INTERNAL : ChatType.EXTERNAL,
flowGraph: getTelemetryFlowObj(nodes, edges)
},
orgId
)
/*** Prepare response ***/
let result: ICommonObject = {}
packages/server/src/utils/buildChatflow.ts
+133 -35
View File
@@ -15,6 +15,7 @@ import {
mapExtToInputField,
getFileFromUpload,
removeSpecificFileFromUpload,
EvaluationRunner,
handleEscapeCharacters
} from 'flowise-components'
import { StatusCodes } from 'http-status-codes'
@@ -22,8 +23,8 @@ import {
IncomingInput,
IMessage,
INodeData,
IReactFlowObject,
IReactFlowNode,
IReactFlowObject,
IDepthQueue,
ChatType,
IChatMessage,
@@ -59,11 +60,15 @@ import {
import { validateChatflowAPIKey } from './validateKey'
import logger from './logger'
import { utilAddChatMessage } from './addChatMesage'
import { checkPredictions, checkStorage, updatePredictionsUsage, updateStorageUsage } from './quotaUsage'
import { buildAgentGraph } from './buildAgentGraph'
import { getErrorMessage } from '../errors/utils'
import { FLOWISE_METRIC_COUNTERS, FLOWISE_COUNTER_STATUS, IMetricsProvider } from '../Interface.Metrics'
import { getWorkspaceSearchOptions } from '../enterprise/utils/ControllerServiceUtils'
import { OMIT_QUEUE_JOB_DATA } from './constants'
import { executeAgentFlow } from './buildAgentflow'
import { Workspace } from '../enterprise/database/entities/workspace.entity'
import { Organization } from '../enterprise/database/entities/organization.entity'
/*
* Initialize the ending node to be executed
@@ -230,15 +235,21 @@ export const executeFlow = async ({
incomingInput,
chatflow,
chatId,
isEvaluation,
evaluationRunId,
appDataSource,
telemetry,
cachePool,
usageCacheManager,
sseStreamer,
baseURL,
isInternal,
files,
signal,
isTool
isTool,
orgId,
workspaceId,
subscriptionId
}: IExecuteFlowParams) => {
// Ensure incomingInput has all required properties with default values
incomingInput = {
@@ -265,6 +276,8 @@ export const executeFlow = async ({
if (uploads) {
fileUploads = uploads
for (let i = 0; i < fileUploads.length; i += 1) {
await checkStorage(orgId, subscriptionId, usageCacheManager)
const upload = fileUploads[i]
// if upload in an image, a rag file, or audio
@@ -273,7 +286,8 @@ export const executeFlow = async ({
const splitDataURI = upload.data.split(',')
const bf = Buffer.from(splitDataURI.pop() || '', 'base64')
const mime = splitDataURI[0].split(':')[1].split(';')[0]
await addSingleFileToStorage(mime, bf, filename, chatflowid, chatId)
const { totalSize } = await addSingleFileToStorage(mime, bf, filename, orgId, chatflowid, chatId)
await updateStorageUsage(orgId, workspaceId, totalSize, usageCacheManager)
upload.type = 'stored-file'
// Omit upload.data since we don't store the content in database
fileUploads[i] = omit(upload, ['data'])
@@ -287,7 +301,7 @@ export const executeFlow = async ({
// Run Speech to Text conversion
if (upload.mime === 'audio/webm' || upload.mime === 'audio/mp4' || upload.mime === 'audio/ogg') {
logger.debug(`Attempting a speech to text conversion...`)
logger.debug(`[server]: [${orgId}]: Attempting a speech to text conversion...`)
let speechToTextConfig: ICommonObject = {}
if (chatflow.speechToText) {
const speechToTextProviders = JSON.parse(chatflow.speechToText)
@@ -302,13 +316,14 @@ export const executeFlow = async ({
}
if (speechToTextConfig) {
const options: ICommonObject = {
orgId,
chatId,
chatflowid,
appDataSource,
databaseEntities: databaseEntities
}
const speechToTextResult = await convertSpeechToText(upload, speechToTextConfig, options)
logger.debug(`Speech to text result: ${speechToTextResult}`)
logger.debug(`[server]: [${orgId}]: Speech to text result: ${speechToTextResult}`)
if (speechToTextResult) {
incomingInput.question = speechToTextResult
question = speechToTextResult
@@ -329,11 +344,21 @@ export const executeFlow = async ({
if (files?.length) {
overrideConfig = { ...incomingInput }
for (const file of files) {
await checkStorage(orgId, subscriptionId, usageCacheManager)
const fileNames: string[] = []
const fileBuffer = await getFileFromUpload(file.path ?? file.key)
// Address file name with special characters: https://github.com/expressjs/multer/issues/1104
file.originalname = Buffer.from(file.originalname, 'latin1').toString('utf8')
const storagePath = await addArrayFilesToStorage(file.mimetype, fileBuffer, file.originalname, fileNames, chatflowid)
const { path: storagePath, totalSize } = await addArrayFilesToStorage(
file.mimetype,
fileBuffer,
file.originalname,
fileNames,
orgId,
chatflowid
)
await updateStorageUsage(orgId, workspaceId, totalSize, usageCacheManager)
const fileInputFieldFromMimeType = mapMimeTypeToInputField(file.mimetype)
@@ -382,16 +407,21 @@ export const executeFlow = async ({
incomingInput,
chatflow,
chatId,
evaluationRunId,
appDataSource,
telemetry,
cachePool,
usageCacheManager,
sseStreamer,
baseURL,
isInternal,
uploadedFilesContent,
fileUploads,
signal,
isTool
isTool,
orgId,
workspaceId,
subscriptionId
})
}
@@ -443,7 +473,7 @@ export const executeFlow = async ({
})
/*** Get API Config ***/
const availableVariables = await appDataSource.getRepository(Variable).find()
const availableVariables = await appDataSource.getRepository(Variable).findBy(getWorkspaceSearchOptions(workspaceId))
const { nodeOverrides, variableOverrides, apiOverrideStatus } = getAPIOverrideConfig(chatflow)
const flowConfig: IFlowConfig = {
@@ -455,7 +485,7 @@ export const executeFlow = async ({
...incomingInput.overrideConfig
}
logger.debug(`[server]: Start building flow ${chatflowid}`)
logger.debug(`[server]: [${orgId}]: Start building flow ${chatflowid}`)
/*** BFS to traverse from Starting Nodes to Ending Node ***/
const reactFlowNodes = await buildFlow({
@@ -479,9 +509,13 @@ export const executeFlow = async ({
availableVariables,
variableOverrides,
cachePool,
usageCacheManager,
isUpsert: false,
uploads,
baseURL
baseURL,
orgId,
workspaceId,
subscriptionId
})
const setVariableNodesOutput = getSetVariableNodesOutput(reactFlowNodes)
@@ -506,7 +540,9 @@ export const executeFlow = async ({
shouldStreamResponse: true, // agentflow is always streamed
cachePool,
baseURL,
signal
signal,
orgId,
workspaceId
})
if (streamResults) {
@@ -556,13 +592,17 @@ export const executeFlow = async ({
}
const chatMessage = await utilAddChatMessage(apiMessage, appDataSource)
await telemetry.sendTelemetry('agentflow_prediction_sent', {
version: await getAppVersion(),
agentflowId: agentflow.id,
chatId,
type: isInternal ? ChatType.INTERNAL : ChatType.EXTERNAL,
flowGraph: getTelemetryFlowObj(nodes, edges)
})
await telemetry.sendTelemetry(
'agentflow_prediction_sent',
{
version: await getAppVersion(),
agentflowId: agentflow.id,
chatId,
type: isInternal ? ChatType.INTERNAL : ChatType.EXTERNAL,
flowGraph: getTelemetryFlowObj(nodes, edges)
},
orgId
)
// Find the previous chat message with the same action id and remove the action
if (incomingInput.action && Object.keys(incomingInput.action).length) {
@@ -596,6 +636,7 @@ export const executeFlow = async ({
// Prepare response
let result: ICommonObject = {}
result.text = finalResult
result.question = incomingInput.question
result.chatId = chatId
result.chatMessageId = chatMessage?.id
@@ -605,7 +646,6 @@ export const executeFlow = async ({
if (finalAction && Object.keys(finalAction).length) result.action = finalAction
if (Object.keys(setVariableNodesOutput).length) result.flowVariables = setVariableNodesOutput
result.followUpPrompts = JSON.stringify(apiMessage.followUpPrompts)
return result
}
return undefined
@@ -643,16 +683,23 @@ export const executeFlow = async ({
/*** Prepare run params ***/
const runParams = {
orgId,
workspaceId,
subscriptionId,
chatId,
chatflowid,
apiMessageId,
logger,
appDataSource,
databaseEntities,
usageCacheManager,
analytic: chatflow.analytic,
uploads,
prependMessages,
...(isStreamValid && { sseStreamer, shouldStreamResponse: isStreamValid })
...(isStreamValid && { sseStreamer, shouldStreamResponse: isStreamValid }),
evaluationRunId,
updateStorageUsage,
checkStorage
}
/*** Run the ending node ***/
@@ -669,7 +716,7 @@ export const executeFlow = async ({
role: 'userMessage',
content: question,
chatflowid,
chatType: isInternal ? ChatType.INTERNAL : ChatType.EXTERNAL,
chatType: isEvaluation ? ChatType.EVALUATION : isInternal ? ChatType.INTERNAL : ChatType.EXTERNAL,
chatId,
memoryType,
sessionId,
@@ -725,7 +772,7 @@ export const executeFlow = async ({
role: 'apiMessage',
content: resultText,
chatflowid,
chatType: isInternal ? ChatType.INTERNAL : ChatType.EXTERNAL,
chatType: isEvaluation ? ChatType.EVALUATION : isInternal ? ChatType.INTERNAL : ChatType.EXTERNAL,
chatId,
memoryType,
sessionId
@@ -749,15 +796,22 @@ export const executeFlow = async ({
const chatMessage = await utilAddChatMessage(apiMessage, appDataSource)
logger.debug(`[server]: Finished running ${endingNodeData.label} (${endingNodeData.id})`)
await telemetry.sendTelemetry('prediction_sent', {
version: await getAppVersion(),
chatflowId: chatflowid,
chatId,
type: isInternal ? ChatType.INTERNAL : ChatType.EXTERNAL,
flowGraph: getTelemetryFlowObj(nodes, edges)
})
logger.debug(`[server]: [${orgId}]: Finished running ${endingNodeData.label} (${endingNodeData.id})`)
if (evaluationRunId) {
const metrics = await EvaluationRunner.getAndDeleteMetrics(evaluationRunId)
result.metrics = metrics
}
await telemetry.sendTelemetry(
'prediction_sent',
{
version: await getAppVersion(),
chatflowId: chatflowid,
chatId,
type: isInternal ? ChatType.INTERNAL : ChatType.EXTERNAL,
flowGraph: getTelemetryFlowObj(nodes, edges)
},
orgId
)
/*** Prepare response ***/
result.question = incomingInput.question // return the question in the response, this is used when input text is empty but question is in audio format
@@ -830,6 +884,7 @@ const checkIfStreamValid = async (
*/
export const utilBuildChatflow = async (req: Request, isInternal: boolean = false): Promise<any> => {
const appServer = getRunningExpressApp()
const chatflowid = req.params.id
// Check if chatflow exists
@@ -841,7 +896,6 @@ export const utilBuildChatflow = async (req: Request, isInternal: boolean = fals
}
const isAgentFlow = chatflow.type === 'MULTIAGENT'
const httpProtocol = req.get('x-forwarded-proto') || req.protocol
const baseURL = `${httpProtocol}://${req.get('host')}`
const incomingInput: IncomingInput = req.body || {} // Ensure incomingInput is never undefined
@@ -849,6 +903,20 @@ export const utilBuildChatflow = async (req: Request, isInternal: boolean = fals
const files = (req.files as Express.Multer.File[]) || []
const abortControllerId = `${chatflow.id}_${chatId}`
const isTool = req.get('flowise-tool') === 'true'
const isEvaluation: boolean = req.headers['X-Flowise-Evaluation'] || req.body.evaluation
let evaluationRunId = ''
if (isEvaluation) {
evaluationRunId = req.body.evaluationRunId
if (evaluationRunId) {
const newEval = {
evaluation: {
status: true,
evaluationRunId
}
}
chatflow.analytic = JSON.stringify(newEval)
}
}
try {
// Validate API Key if its external API request
@@ -859,6 +927,28 @@ export const utilBuildChatflow = async (req: Request, isInternal: boolean = fals
}
}
// This can be public API, so we can only get orgId from the chatflow
const chatflowWorkspaceId = chatflow.workspaceId
const workspace = await appServer.AppDataSource.getRepository(Workspace).findOneBy({
id: chatflowWorkspaceId
})
if (!workspace) {
throw new InternalFlowiseError(StatusCodes.NOT_FOUND, `Workspace ${chatflowWorkspaceId} not found`)
}
const workspaceId = workspace.id
const org = await appServer.AppDataSource.getRepository(Organization).findOneBy({
id: workspace.organizationId
})
if (!org) {
throw new InternalFlowiseError(StatusCodes.NOT_FOUND, `Organization ${workspace.organizationId} not found`)
}
const orgId = org.id
const subscriptionId = org.subscriptionId as string
await checkPredictions(orgId, subscriptionId, appServer.usageCacheManager)
const executeData: IExecuteFlowParams = {
incomingInput, // Use the defensively created incomingInput variable
chatflow,
@@ -866,18 +956,24 @@ export const utilBuildChatflow = async (req: Request, isInternal: boolean = fals
baseURL,
isInternal,
files,
isEvaluation,
evaluationRunId,
appDataSource: appServer.AppDataSource,
sseStreamer: appServer.sseStreamer,
telemetry: appServer.telemetry,
cachePool: appServer.cachePool,
componentNodes: appServer.nodesPool.componentNodes,
isTool // used to disable streaming if incoming request its from ChatflowTool
isTool, // used to disable streaming if incoming request its from ChatflowTool
usageCacheManager: appServer.usageCacheManager,
orgId,
workspaceId,
subscriptionId
}
if (process.env.MODE === MODE.QUEUE) {
const predictionQueue = appServer.queueManager.getQueue('prediction')
const job = await predictionQueue.addJob(omit(executeData, OMIT_QUEUE_JOB_DATA))
logger.debug(`[server]: Job added to queue: ${job.id}`)
logger.debug(`[server]: [${orgId}]: Job added to queue: ${job.id}`)
const queueEvents = predictionQueue.getQueueEvents()
const result = await job.waitUntilFinished(queueEvents)
@@ -885,7 +981,7 @@ export const utilBuildChatflow = async (req: Request, isInternal: boolean = fals
if (!result) {
throw new Error('Job execution failed')
}
await updatePredictionsUsage(orgId, subscriptionId, workspaceId, appServer.usageCacheManager)
incrementSuccessMetricCounter(appServer.metricsProvider, isInternal, isAgentFlow)
return result
} else {
@@ -893,9 +989,11 @@ export const utilBuildChatflow = async (req: Request, isInternal: boolean = fals
const signal = new AbortController()
appServer.abortControllerPool.add(abortControllerId, signal)
executeData.signal = signal
const result = await executeFlow(executeData)
appServer.abortControllerPool.remove(abortControllerId)
await updatePredictionsUsage(orgId, subscriptionId, workspaceId, appServer.usageCacheManager)
incrementSuccessMetricCounter(appServer.metricsProvider, isInternal, isAgentFlow)
return result
}
packages/server/src/utils/constants.ts
+70 -2
View File
@@ -1,3 +1,8 @@
import Auth0SSO from '../enterprise/sso/Auth0SSO'
import AzureSSO from '../enterprise/sso/AzureSSO'
import GithubSSO from '../enterprise/sso/GithubSSO'
import GoogleSSO from '../enterprise/sso/GoogleSSO'
export const WHITELIST_URLS = [
'/api/v1/verify/apikey/',
'/api/v1/chatflows/apikey/',
@@ -19,10 +24,63 @@ export const WHITELIST_URLS = [
'/api/v1/version',
'/api/v1/attachments',
'/api/v1/metrics',
'/api/v1/nvidia-nim'
'/api/v1/nvidia-nim',
'/api/v1/auth/resolve',
'/api/v1/auth/login',
'/api/v1/auth/refreshToken',
'/api/v1/settings',
'/api/v1/account/logout',
'/api/v1/account/verify',
'/api/v1/account/register',
'/api/v1/account/resend-verification',
'/api/v1/account/forgot-password',
'/api/v1/account/reset-password',
'/api/v1/account/basic-auth',
'/api/v1/loginmethod',
'/api/v1/pricing',
'/api/v1/user/test',
AzureSSO.LOGIN_URI,
AzureSSO.LOGOUT_URI,
AzureSSO.CALLBACK_URI,
GoogleSSO.LOGIN_URI,
GoogleSSO.LOGOUT_URI,
GoogleSSO.CALLBACK_URI,
Auth0SSO.LOGIN_URI,
Auth0SSO.LOGOUT_URI,
Auth0SSO.CALLBACK_URI,
GithubSSO.LOGIN_URI,
GithubSSO.LOGOUT_URI,
GithubSSO.CALLBACK_URI
]
export const OMIT_QUEUE_JOB_DATA = ['componentNodes', 'appDataSource', 'sseStreamer', 'telemetry', 'cachePool']
export const enum GeneralErrorMessage {
UNAUTHORIZED = 'Unauthorized',
UNHANDLED_EDGE_CASE = 'Unhandled Edge Case',
INVALID_PASSWORD = 'Invalid Password',
NOT_ALLOWED_TO_DELETE_OWNER = 'Not Allowed To Delete Owner',
INTERNAL_SERVER_ERROR = 'Internal Server Error'
}
export const enum GeneralSuccessMessage {
CREATED = 'Resource Created Successful',
UPDATED = 'Resource Updated Successful',
DELETED = 'Resource Deleted Successful',
FETCHED = 'Resource Fetched Successful',
LOGGED_IN = 'Login Successful',
LOGGED_OUT = 'Logout Successful'
}
export const DOCUMENT_STORE_BASE_FOLDER = 'docustore'
export const OMIT_QUEUE_JOB_DATA = [
'componentNodes',
'appDataSource',
'sseStreamer',
'telemetry',
'cachePool',
'usageCacheManager',
'abortControllerPool'
]
export const INPUT_PARAMS_TYPE = [
'asyncOptions',
@@ -42,3 +100,13 @@ export const INPUT_PARAMS_TYPE = [
'folder',
'tabs'
]
export const LICENSE_QUOTAS = {
// Renew per month
PREDICTIONS_LIMIT: 'quota:predictions',
// Static
FLOWS_LIMIT: 'quota:flows',
USERS_LIMIT: 'quota:users',
STORAGE_LIMIT: 'quota:storage',
ADDITIONAL_SEATS_LIMIT: 'quota:additionalSeats'
} as const
packages/server/src/utils/createAttachment.ts
+43 -4
View File
@@ -12,9 +12,12 @@ import {
} from 'flowise-components'
import { getRunningExpressApp } from './getRunningExpressApp'
import { getErrorMessage } from '../errors/utils'
import { checkStorage, updateStorageUsage } from './quotaUsage'
import { ChatFlow } from '../database/entities/ChatFlow'
import { Workspace } from '../enterprise/database/entities/workspace.entity'
import { Organization } from '../enterprise/database/entities/organization.entity'
import { InternalFlowiseError } from '../errors/internalFlowiseError'
import { StatusCodes } from 'http-status-codes'
import { ChatFlow } from '../database/entities/ChatFlow'
/**
* Create attachment
@@ -46,6 +49,32 @@ export const createFileAttachment = async (req: Request) => {
throw new InternalFlowiseError(StatusCodes.NOT_FOUND, `Chatflow ${chatflowid} not found`)
}
let orgId = req.user?.activeOrganizationId || ''
let workspaceId = req.user?.activeWorkspaceId || ''
let subscriptionId = req.user?.activeOrganizationSubscriptionId || ''
// This is one of the WHITELIST_URLS, API can be public and there might be no req.user
if (!orgId || !workspaceId) {
const chatflowWorkspaceId = chatflow.workspaceId
const workspace = await appServer.AppDataSource.getRepository(Workspace).findOneBy({
id: chatflowWorkspaceId
})
if (!workspace) {
throw new InternalFlowiseError(StatusCodes.NOT_FOUND, `Workspace ${chatflowWorkspaceId} not found`)
}
workspaceId = workspace.id
const org = await appServer.AppDataSource.getRepository(Organization).findOneBy({
id: workspace.organizationId
})
if (!org) {
throw new InternalFlowiseError(StatusCodes.NOT_FOUND, `Organization ${workspace.organizationId} not found`)
}
orgId = org.id
subscriptionId = org.subscriptionId as string
}
// Parse chatbot configuration to get file upload settings
let pdfConfig = {
usage: 'perPage',
@@ -75,6 +104,7 @@ export const createFileAttachment = async (req: Request) => {
const fileLoaderNodeInstance = new fileLoaderNodeModule.nodeClass()
const options = {
retrieveAttachmentChatId: true,
orgId,
chatflowid,
chatId
}
@@ -83,13 +113,22 @@ export const createFileAttachment = async (req: Request) => {
if (files.length) {
const isBase64 = req.body.base64
for (const file of files) {
await checkStorage(orgId, subscriptionId, appServer.usageCacheManager)
const fileBuffer = await getFileFromUpload(file.path ?? file.key)
const fileNames: string[] = []
// Address file name with special characters: https://github.com/expressjs/multer/issues/1104
file.originalname = Buffer.from(file.originalname, 'latin1').toString('utf8')
const storagePath = await addArrayFilesToStorage(file.mimetype, fileBuffer, file.originalname, fileNames, chatflowid, chatId)
const { path: storagePath, totalSize } = await addArrayFilesToStorage(
file.mimetype,
fileBuffer,
file.originalname,
fileNames,
orgId,
chatflowid,
chatId
)
await updateStorageUsage(orgId, workspaceId, totalSize, appServer.usageCacheManager)
const fileInputFieldFromMimeType = mapMimeTypeToInputField(file.mimetype)
packages/server/src/utils/fileRepository.ts
+21 -6
View File
@@ -1,6 +1,8 @@
import { ChatFlow } from '../database/entities/ChatFlow'
import { IReactFlowObject } from '../Interface'
import { addBase64FilesToStorage } from 'flowise-components'
import { checkStorage, updateStorageUsage } from './quotaUsage'
import { UsageCacheManager } from '../UsageCacheManager'
export const containsBase64File = (chatflow: ChatFlow) => {
const parsedFlowData: IReactFlowObject = JSON.parse(chatflow.flowData)
@@ -46,11 +48,19 @@ export const containsBase64File = (chatflow: ChatFlow) => {
return found
}
export const updateFlowDataWithFilePaths = async (chatflowid: string, flowData: string) => {
export const updateFlowDataWithFilePaths = async (
chatflowid: string,
flowData: string,
orgId: string,
workspaceId: string,
subscriptionId: string,
usageCacheManager: UsageCacheManager
) => {
try {
const parsedFlowData: IReactFlowObject = JSON.parse(flowData)
const re = new RegExp('^data.*;base64', 'i')
const nodes = parsedFlowData.nodes
for (let j = 0; j < nodes.length; j++) {
const node = nodes[j]
if (node.data.category !== 'Document Loaders') {
@@ -75,21 +85,26 @@ export const updateFlowDataWithFilePaths = async (chatflowid: string, flowData:
for (let j = 0; j < files.length; j++) {
const file = files[j]
if (re.test(file)) {
node.data.inputs[key] = await addBase64FilesToStorage(file, chatflowid, fileNames)
await checkStorage(orgId, subscriptionId, usageCacheManager)
const { path, totalSize } = await addBase64FilesToStorage(file, chatflowid, fileNames, orgId)
node.data.inputs[key] = path
await updateStorageUsage(orgId, workspaceId, totalSize, usageCacheManager)
}
}
} catch (e) {
continue
}
} else if (re.test(input)) {
node.data.inputs[key] = await addBase64FilesToStorage(input, chatflowid, fileNames)
await checkStorage(orgId, subscriptionId, usageCacheManager)
const { path, totalSize } = await addBase64FilesToStorage(input, chatflowid, fileNames, orgId)
node.data.inputs[key] = path
await updateStorageUsage(orgId, workspaceId, totalSize, usageCacheManager)
}
}
}
}
return JSON.stringify(parsedFlowData)
} catch (e) {
return ''
} catch (e: any) {
throw new Error(`Error updating flow data with file paths: ${e.message}`)
}
}
packages/server/src/utils/getChatMessage.ts
+18 -2
View File
@@ -2,6 +2,7 @@ import { MoreThanOrEqual, LessThanOrEqual, Between, In } from 'typeorm'
import { ChatMessageRatingType, ChatType } from '../Interface'
import { ChatMessage } from '../database/entities/ChatMessage'
import { ChatMessageFeedback } from '../database/entities/ChatMessageFeedback'
import { ChatFlow } from '../database/entities/ChatFlow'
import { getRunningExpressApp } from '../utils/getRunningExpressApp'
import { aMonthAgo } from '.'
@@ -30,6 +31,7 @@ interface GetChatMessageParams {
messageId?: string
feedback?: boolean
feedbackTypes?: ChatMessageRatingType[]
activeWorkspaceId?: string
}
export const utilGetChatMessage = async ({
@@ -43,10 +45,21 @@ export const utilGetChatMessage = async ({
endDate,
messageId,
feedback,
feedbackTypes
feedbackTypes,
activeWorkspaceId
}: GetChatMessageParams): Promise<ChatMessage[]> => {
const appServer = getRunningExpressApp()
// Check if chatflow workspaceId is same as activeWorkspaceId
if (activeWorkspaceId) {
const chatflow = await appServer.AppDataSource.getRepository(ChatFlow).findOneBy({
id: chatflowid
})
if (chatflow?.workspaceId !== activeWorkspaceId) {
throw new Error('Unauthorized access')
}
}
if (feedback) {
const query = await appServer.AppDataSource.getRepository(ChatMessage).createQueryBuilder('chat_message')
@@ -102,6 +115,7 @@ export const utilGetChatMessage = async ({
}
let createdDateQuery
if (startDate || endDate) {
if (startDate && endDate) {
createdDateQuery = Between(new Date(startDate), new Date(endDate))
@@ -112,7 +126,7 @@ export const utilGetChatMessage = async ({
}
}
return await appServer.AppDataSource.getRepository(ChatMessage).find({
const messages = await appServer.AppDataSource.getRepository(ChatMessage).find({
where: {
chatflowid,
chatType: chatTypes?.length ? In(chatTypes) : undefined,
@@ -129,4 +143,6 @@ export const utilGetChatMessage = async ({
createdDate: sortOrder === 'DESC' ? 'DESC' : 'ASC'
}
})
return messages
}
packages/server/src/utils/index.ts
+55 -52
View File
@@ -5,6 +5,7 @@
import path from 'path'
import fs from 'fs'
import logger from './logger'
import { v4 as uuidv4 } from 'uuid'
import {
IChatFlow,
IComponentCredentials,
@@ -63,6 +64,8 @@ import {
SecretsManagerClient,
SecretsManagerClientConfig
} from '@aws-sdk/client-secrets-manager'
import { checkStorage, updateStorageUsage } from './quotaUsage'
import { UsageCacheManager } from '../UsageCacheManager'
export const QUESTION_VAR_PREFIX = 'question'
export const FILE_ATTACHMENT_PREFIX = 'file_attachment'
@@ -203,6 +206,22 @@ export const constructGraphs = (
return { graph, nodeDependencies }
}
/**
* Get starting node and check if flow is valid
* @param {INodeDependencies} nodeDependencies
*/
export const getStartingNode = (nodeDependencies: INodeDependencies) => {
// Find starting node
const startingNodeIds = [] as string[]
Object.keys(nodeDependencies).forEach((nodeId) => {
if (nodeDependencies[nodeId] === 0) {
startingNodeIds.push(nodeId)
}
})
return { startingNodeIds }
}
/**
* Get starting nodes and check if flow is valid
* @param {INodeDependencies} graph
@@ -239,22 +258,6 @@ export const getStartingNodes = (graph: INodeDirectedGraph, endNodeId: string) =
return { startingNodeIds, depthQueue: depthQueueReversed }
}
/**
* Get starting node and check if flow is valid
* @param {INodeDependencies} nodeDependencies
*/
export const getStartingNode = (nodeDependencies: INodeDependencies) => {
// Find starting node
const startingNodeIds = [] as string[]
Object.keys(nodeDependencies).forEach((nodeId) => {
if (nodeDependencies[nodeId] === 0) {
startingNodeIds.push(nodeId)
}
})
return { startingNodeIds }
}
/**
* Get all connected nodes from startnode
* @param {INodeDependencies} graph
@@ -497,6 +500,10 @@ type BuildFlowParams = {
stopNodeId?: string
uploads?: IFileUpload[]
baseURL?: string
orgId?: string
workspaceId?: string
subscriptionId?: string
usageCacheManager?: UsageCacheManager
uploadedFilesContent?: string
}
@@ -528,7 +535,11 @@ export const buildFlow = async ({
isUpsert,
stopNodeId,
uploads,
baseURL
baseURL,
orgId,
workspaceId,
subscriptionId,
usageCacheManager
}: BuildFlowParams) => {
const flowNodes = cloneDeep(reactFlowNodes)
@@ -591,8 +602,11 @@ export const buildFlow = async ({
)
if (isUpsert && stopNodeId && nodeId === stopNodeId) {
logger.debug(`[server]: Upserting ${reactFlowNode.data.label} (${reactFlowNode.data.id})`)
logger.debug(`[server]: [${orgId}]: Upserting ${reactFlowNode.data.label} (${reactFlowNode.data.id})`)
const indexResult = await newNodeInstance.vectorStoreMethods!['upsert']!.call(newNodeInstance, reactFlowNodeData, {
orgId,
workspaceId,
subscriptionId,
chatId,
sessionId,
chatflowid,
@@ -602,12 +616,13 @@ export const buildFlow = async ({
appDataSource,
databaseEntities,
cachePool,
usageCacheManager,
dynamicVariables,
uploads,
baseURL
})
if (indexResult) upsertHistory['result'] = indexResult
logger.debug(`[server]: Finished upserting ${reactFlowNode.data.label} (${reactFlowNode.data.id})`)
logger.debug(`[server]: [${orgId}]: Finished upserting ${reactFlowNode.data.label} (${reactFlowNode.data.id})`)
break
} else if (
!isUpsert &&
@@ -616,9 +631,12 @@ export const buildFlow = async ({
) {
initializedNodes.add(nodeId)
} else {
logger.debug(`[server]: Initializing ${reactFlowNode.data.label} (${reactFlowNode.data.id})`)
logger.debug(`[server]: [${orgId}]: Initializing ${reactFlowNode.data.label} (${reactFlowNode.data.id})`)
const finalQuestion = uploadedFilesContent ? `${uploadedFilesContent}\n\n${question}` : question
let outputResult = await newNodeInstance.init(reactFlowNodeData, finalQuestion, {
orgId,
workspaceId,
subscriptionId,
chatId,
sessionId,
chatflowid,
@@ -627,11 +645,14 @@ export const buildFlow = async ({
appDataSource,
databaseEntities,
cachePool,
usageCacheManager,
isUpsert,
dynamicVariables,
uploads,
baseURL,
componentNodes: componentNodes as ICommonObject
componentNodes,
updateStorageUsage,
checkStorage
})
// Save dynamic variables
@@ -676,11 +697,11 @@ export const buildFlow = async ({
flowNodes[nodeIndex].data.instance = outputResult
logger.debug(`[server]: Finished initializing ${reactFlowNode.data.label} (${reactFlowNode.data.id})`)
logger.debug(`[server]: [${orgId}]: Finished initializing ${reactFlowNode.data.label} (${reactFlowNode.data.id})`)
initializedNodes.add(reactFlowNode.data.id)
}
} catch (e: any) {
logger.error(e)
logger.error(`[server]: [${orgId}]:`, e)
throw new Error(e)
}
@@ -744,6 +765,7 @@ export const clearSessionMemory = async (
componentNodes: IComponentNodes,
chatId: string,
appDataSource: DataSource,
orgId?: string,
sessionId?: string,
memoryType?: string,
isClearFromViewMessageDialog?: string
@@ -757,7 +779,7 @@ export const clearSessionMemory = async (
const nodeInstanceFilePath = componentNodes[node.data.name].filePath as string
const nodeModule = await import(nodeInstanceFilePath)
const newNodeInstance = new nodeModule.nodeClass()
const options: ICommonObject = { chatId, appDataSource, databaseEntities, logger }
const options: ICommonObject = { orgId, chatId, appDataSource, databaseEntities, logger }
// SessionId always take priority first because it is the sessionId used for 3rd party memory node
if (sessionId && node.data.inputs) {
@@ -1261,7 +1283,6 @@ export const findAvailableConfigs = (reactFlowNodes: IReactFlowNode[], component
for (const flowNode of reactFlowNodes) {
for (const inputParam of flowNode.data.inputParams) {
let obj: IOverrideConfig | undefined
if (inputParam.type === 'file') {
obj = {
node: flowNode.data.label,
@@ -1500,7 +1521,6 @@ export const decryptCredentialData = async (
if (USE_AWS_SECRETS_MANAGER && secretsManagerClient) {
try {
logger.info(`[server]: Reading AWS Secret: ${encryptedData}`)
if (encryptedData.startsWith('FlowiseCredential_')) {
const command = new GetSecretValueCommand({ SecretId: encryptedData })
const response = await secretsManagerClient.send(command)
@@ -1567,6 +1587,10 @@ export const transformToCredentialEntity = async (body: ICredentialReqBody): Pro
const newCredential = new Credential()
Object.assign(newCredential, credentialBody)
if (body.workspaceId) {
newCredential.workspaceId = body.workspaceId
}
return newCredential
}
@@ -1734,21 +1758,6 @@ export const getTelemetryFlowObj = (nodes: IReactFlowNode[], edges: IReactFlowEd
return { nodes: nodeData, edges: edgeData }
}
/**
* Get user settings file
* TODO: move env variables to settings json file, easier configuration
*/
export const getUserSettingsFilePath = () => {
if (process.env.SECRETKEY_PATH) return path.join(process.env.SECRETKEY_PATH, 'settings.json')
const checkPaths = [path.join(getUserHome(), '.flowise', 'settings.json')]
for (const checkPath of checkPaths) {
if (fs.existsSync(checkPath)) {
return checkPath
}
}
return ''
}
/**
* Get app current version
*/
@@ -1815,14 +1824,8 @@ export const getUploadPath = (): string => {
: path.join(getUserHome(), '.flowise', 'uploads')
}
const getOrgId = () => {
const settingsContent = fs.readFileSync(getUserSettingsFilePath(), 'utf8')
try {
const settings = JSON.parse(settingsContent)
return settings.instanceId
} catch (error) {
return ''
}
export function generateId() {
return uuidv4()
}
export const getMulterStorage = () => {
@@ -1837,10 +1840,10 @@ export const getMulterStorage = () => {
s3: s3Client,
bucket: Bucket,
metadata: function (req, file, cb) {
cb(null, { fieldName: file.fieldname, originalName: file.originalname, orgId: getOrgId() })
cb(null, { fieldName: file.fieldname, originalName: file.originalname })
},
key: function (req, file, cb) {
cb(null, `${getOrgId()}/${Date.now().toString()}`)
cb(null, `${generateId()}`)
}
})
})
@@ -1852,7 +1855,7 @@ export const getMulterStorage = () => {
bucket: process.env.GOOGLE_CLOUD_STORAGE_BUCKET_NAME,
keyFilename: process.env.GOOGLE_CLOUD_STORAGE_CREDENTIAL,
uniformBucketLevelAccess: Boolean(process.env.GOOGLE_CLOUD_UNIFORM_BUCKET_ACCESS) ?? true,
destination: `uploads/${getOrgId()}`
destination: `uploads/${generateId()}`
})
})
} else {
packages/server/src/utils/logger.ts
+16 -21
View File
@@ -4,6 +4,7 @@ import { hostname } from 'node:os'
import config from './config' // should be replaced by node-config or similar
import { createLogger, transports, format } from 'winston'
import { NextFunction, Request, Response } from 'express'
import DailyRotateFile from 'winston-daily-rotate-file'
import { S3ClientConfig } from '@aws-sdk/client-s3'
import { LoggingWinston } from '@google-cloud/logging-winston'
@@ -114,13 +115,11 @@ const logger = createLogger({
new transports.Console(),
...(!process.env.STORAGE_TYPE || process.env.STORAGE_TYPE === 'local'
? [
new transports.File({
filename: path.join(logDir, config.logging.server.filename ?? 'server.log'),
new DailyRotateFile({
filename: path.join(logDir, config.logging.server.filename ?? 'server-%DATE%.log'),
datePattern: 'YYYY-MM-DD-HH',
maxSize: '20m',
level: config.logging.server.level ?? 'info'
}),
new transports.File({
filename: path.join(logDir, config.logging.server.errorFilename ?? 'server-error.log'),
level: 'error' // Log only errors to this file
})
]
: []),
@@ -134,13 +133,7 @@ const logger = createLogger({
...(process.env.STORAGE_TYPE === 'gcs' ? [gcsServerStream] : [])
],
exceptionHandlers: [
...(!process.env.STORAGE_TYPE || process.env.STORAGE_TYPE === 'local'
? [
new transports.File({
filename: path.join(logDir, config.logging.server.errorFilename ?? 'server-error.log')
})
]
: []),
...(process.env.DEBUG && process.env.DEBUG === 'true' ? [new transports.Console()] : []),
...(process.env.STORAGE_TYPE === 's3'
? [
new transports.Stream({
@@ -151,13 +144,7 @@ const logger = createLogger({
...(process.env.STORAGE_TYPE === 'gcs' ? [gcsErrorStream] : [])
],
rejectionHandlers: [
...(!process.env.STORAGE_TYPE || process.env.STORAGE_TYPE === 'local'
? [
new transports.File({
filename: path.join(logDir, config.logging.server.errorFilename ?? 'server-error.log')
})
]
: []),
...(process.env.DEBUG && process.env.DEBUG === 'true' ? [new transports.Console()] : []),
...(process.env.STORAGE_TYPE === 's3'
? [
new transports.Stream({
@@ -171,7 +158,14 @@ const logger = createLogger({
export function expressRequestLogger(req: Request, res: Response, next: NextFunction): void {
const unwantedLogURLs = ['/api/v1/node-icon/', '/api/v1/components-credentials-icon/', '/api/v1/ping']
if (/\/api\/v1\//i.test(req.url) && !unwantedLogURLs.some((url) => new RegExp(url, 'i').test(req.url))) {
// Create a sanitized copy of the request body
const sanitizedBody = { ...req.body }
if (sanitizedBody.password) {
sanitizedBody.password = '********'
}
const fileLogger = createLogger({
format: combine(timestamp({ format: 'YYYY-MM-DD HH:mm:ss' }), format.json(), errors({ stack: true })),
defaultMeta: {
@@ -179,13 +173,14 @@ export function expressRequestLogger(req: Request, res: Response, next: NextFunc
request: {
method: req.method,
url: req.url,
body: req.body,
body: sanitizedBody, // Use sanitized body instead of raw body
query: req.query,
params: req.params,
headers: req.headers
}
},
transports: [
...(process.env.DEBUG && process.env.DEBUG === 'true' ? [new transports.Console()] : []),
...(!process.env.STORAGE_TYPE || process.env.STORAGE_TYPE === 'local'
? [
new transports.File({
packages/server/src/utils/quotaUsage.ts
+171
View File
@@ -0,0 +1,171 @@
import { StatusCodes } from 'http-status-codes'
import { InternalFlowiseError } from '../errors/internalFlowiseError'
import { UsageCacheManager } from '../UsageCacheManager'
import { LICENSE_QUOTAS } from './constants'
import logger from './logger'
type UsageType = 'flows' | 'users'
export const ENTERPRISE_FEATURE_FLAGS = [
//'feat:account', // Only for Cloud
'feat:datasets',
'feat:evaluations',
'feat:evaluators',
'feat:files',
'feat:login-activity',
'feat:users',
'feat:workspaces',
'feat:logs',
'feat:roles',
'feat:sso-config'
]
export const getCurrentUsage = async (orgId: string, subscriptionId: string, usageCacheManager: UsageCacheManager) => {
try {
if (!usageCacheManager || !subscriptionId || !orgId) return
const currentStorageUsage = (await usageCacheManager.get(`storage:${orgId}`)) || 0
const currentPredictionsUsage = (await usageCacheManager.get(`predictions:${orgId}`)) || 0
const quotas = await usageCacheManager.getQuotas(subscriptionId)
const storageLimit = quotas[LICENSE_QUOTAS.STORAGE_LIMIT]
const predLimit = quotas[LICENSE_QUOTAS.PREDICTIONS_LIMIT]
return {
predictions: {
usage: currentPredictionsUsage,
limit: predLimit
},
storage: {
usage: currentStorageUsage,
limit: storageLimit
}
}
} catch (error) {
logger.error(`[getCurrentUsage] Error getting usage: ${error}`)
throw error
}
}
// For usage that doesn't renew per month, we just get the count from database and check
export const checkUsageLimit = async (
type: UsageType,
subscriptionId: string,
usageCacheManager: UsageCacheManager,
currentUsage: number
) => {
if (!usageCacheManager || !subscriptionId) return
const quotas = await usageCacheManager.getQuotas(subscriptionId)
let limit = -1
switch (type) {
case 'flows':
limit = quotas[LICENSE_QUOTAS.FLOWS_LIMIT]
break
case 'users':
limit = quotas[LICENSE_QUOTAS.USERS_LIMIT] + (Math.max(quotas[LICENSE_QUOTAS.ADDITIONAL_SEATS_LIMIT], 0) || 0)
break
}
if (limit === -1) return
if (currentUsage > limit) {
throw new InternalFlowiseError(StatusCodes.TOO_MANY_REQUESTS, `Limit exceeded: ${type}`)
}
}
// As predictions limit renew per month, we set to cache with 1 month TTL
export const updatePredictionsUsage = async (
orgId: string,
subscriptionId: string,
_: string = '',
usageCacheManager?: UsageCacheManager
) => {
if (!usageCacheManager) return
const quotas = await usageCacheManager.getQuotas(subscriptionId)
const predictionsLimit = quotas[LICENSE_QUOTAS.PREDICTIONS_LIMIT]
let currentPredictions = 0
const existingPredictions = await usageCacheManager.get(`predictions:${orgId}`)
if (existingPredictions) {
currentPredictions = 1 + (existingPredictions as number) > predictionsLimit ? predictionsLimit : 1 + (existingPredictions as number)
} else {
currentPredictions = 1
}
const currentTTL = await usageCacheManager.getTTL(`predictions:${orgId}`)
if (currentTTL) {
const currentTimestamp = Date.now()
const timeLeft = currentTTL - currentTimestamp
usageCacheManager.set(`predictions:${orgId}`, currentPredictions, timeLeft)
} else {
const subscriptionDetails = await usageCacheManager.getSubscriptionDetails(subscriptionId)
if (subscriptionDetails && subscriptionDetails.created) {
const MS_PER_DAY = 24 * 60 * 60 * 1000
const DAYS = 30
const approximateMonthMs = DAYS * MS_PER_DAY
// Calculate time elapsed since subscription creation
const createdTimestamp = subscriptionDetails.created * 1000 // Convert to milliseconds if timestamp is in seconds
const currentTimestamp = Date.now()
const timeElapsed = currentTimestamp - createdTimestamp
// Calculate remaining time in the current month period
const timeLeft = approximateMonthMs - (timeElapsed % approximateMonthMs)
usageCacheManager.set(`predictions:${orgId}`, currentPredictions, timeLeft)
} else {
// Fallback to default 30 days if no creation date
const MS_PER_DAY = 24 * 60 * 60 * 1000
const DAYS = 30
const approximateMonthMs = DAYS * MS_PER_DAY
usageCacheManager.set(`predictions:${orgId}`, currentPredictions, approximateMonthMs)
}
}
}
export const checkPredictions = async (orgId: string, subscriptionId: string, usageCacheManager: UsageCacheManager) => {
if (!usageCacheManager || !subscriptionId) return
const currentPredictions: number = (await usageCacheManager.get(`predictions:${orgId}`)) || 0
const quotas = await usageCacheManager.getQuotas(subscriptionId)
const predictionsLimit = quotas[LICENSE_QUOTAS.PREDICTIONS_LIMIT]
if (predictionsLimit === -1) return
if (currentPredictions >= predictionsLimit) {
throw new InternalFlowiseError(StatusCodes.TOO_MANY_REQUESTS, 'Predictions limit exceeded')
}
return {
usage: currentPredictions,
limit: predictionsLimit
}
}
// Storage does not renew per month nor do we store the total size in database, so we just store the total size in cache
export const updateStorageUsage = (orgId: string, _: string = '', totalSize: number, usageCacheManager?: UsageCacheManager) => {
if (!usageCacheManager) return
usageCacheManager.set(`storage:${orgId}`, totalSize)
}
export const checkStorage = async (orgId: string, subscriptionId: string, usageCacheManager: UsageCacheManager) => {
if (!usageCacheManager || !subscriptionId) return
let currentStorageUsage = 0
currentStorageUsage = (await usageCacheManager.get(`storage:${orgId}`)) || 0
const quotas = await usageCacheManager.getQuotas(subscriptionId)
const storageLimit = quotas[LICENSE_QUOTAS.STORAGE_LIMIT]
if (storageLimit === -1) return
if (currentStorageUsage >= storageLimit) {
throw new InternalFlowiseError(StatusCodes.TOO_MANY_REQUESTS, 'Storage limit exceeded')
}
return {
usage: currentStorageUsage,
limit: storageLimit
}
}
packages/server/src/utils/telemetry.ts
+9 -23
View File
@@ -1,8 +1,11 @@
import { v4 as uuidv4 } from 'uuid'
import { PostHog } from 'posthog-node'
import path from 'path'
import fs from 'fs'
import { getUserHome, getUserSettingsFilePath } from '.'
import { getAppVersion } from '../utils'
export enum TelemetryEventType {
'USER_CREATED' = 'user_created',
'ORGANIZATION_CREATED' = 'organization_created'
}
export class Telemetry {
postHog?: PostHog
@@ -15,27 +18,10 @@ export class Telemetry {
}
}
async id(): Promise<string> {
try {
const settingsContent = await fs.promises.readFile(getUserSettingsFilePath(), 'utf8')
const settings = JSON.parse(settingsContent)
return settings.instanceId
} catch (error) {
const instanceId = uuidv4()
const settings = {
instanceId
}
const defaultLocation = process.env.SECRETKEY_PATH
? path.join(process.env.SECRETKEY_PATH, 'settings.json')
: path.join(getUserHome(), '.flowise', 'settings.json')
await fs.promises.writeFile(defaultLocation, JSON.stringify(settings, null, 2))
return instanceId
}
}
async sendTelemetry(event: string, properties = {}): Promise<void> {
async sendTelemetry(event: string, properties: Record<string, any> = {}, orgId = ''): Promise<void> {
properties.version = await getAppVersion()
if (this.postHog) {
const distinctId = await this.id()
const distinctId = orgId || uuidv4()
this.postHog.capture({
event,
distinctId,
packages/server/src/utils/upsertVector.ts
+66 -15
View File
@@ -28,11 +28,15 @@ import { getRunningExpressApp } from '../utils/getRunningExpressApp'
import { UpsertHistory } from '../database/entities/UpsertHistory'
import { InternalFlowiseError } from '../errors/internalFlowiseError'
import { StatusCodes } from 'http-status-codes'
import { checkStorage, updateStorageUsage } from './quotaUsage'
import { getErrorMessage } from '../errors/utils'
import { v4 as uuidv4 } from 'uuid'
import { FLOWISE_COUNTER_STATUS, FLOWISE_METRIC_COUNTERS } from '../Interface.Metrics'
import { Variable } from '../database/entities/Variable'
import { getWorkspaceSearchOptions } from '../enterprise/utils/ControllerServiceUtils'
import { OMIT_QUEUE_JOB_DATA } from './constants'
import { Workspace } from '../enterprise/database/entities/workspace.entity'
import { Organization } from '../enterprise/database/entities/organization.entity'
export const executeUpsert = async ({
componentNodes,
@@ -43,7 +47,11 @@ export const executeUpsert = async ({
telemetry,
cachePool,
isInternal,
files
files,
orgId,
workspaceId,
subscriptionId,
usageCacheManager
}: IExecuteFlowParams) => {
const question = incomingInput.question
let overrideConfig = incomingInput.overrideConfig ?? {}
@@ -56,11 +64,21 @@ export const executeUpsert = async ({
if (files?.length) {
overrideConfig = { ...incomingInput }
for (const file of files) {
await checkStorage(orgId, subscriptionId, usageCacheManager)
const fileNames: string[] = []
const fileBuffer = await getFileFromUpload(file.path ?? file.key)
// Address file name with special characters: https://github.com/expressjs/multer/issues/1104
file.originalname = Buffer.from(file.originalname, 'latin1').toString('utf8')
const storagePath = await addArrayFilesToStorage(file.mimetype, fileBuffer, file.originalname, fileNames, chatflowid)
const { path: storagePath, totalSize } = await addArrayFilesToStorage(
file.mimetype,
fileBuffer,
file.originalname,
fileNames,
orgId,
chatflowid
)
await updateStorageUsage(orgId, workspaceId, totalSize, usageCacheManager)
const fileInputFieldFromMimeType = mapMimeTypeToInputField(file.mimetype)
@@ -147,7 +165,7 @@ export const executeUpsert = async ({
const { startingNodeIds, depthQueue } = getStartingNodes(filteredGraph, stopNodeId)
/*** Get API Config ***/
const availableVariables = await appDataSource.getRepository(Variable).find()
const availableVariables = await appDataSource.getRepository(Variable).findBy(getWorkspaceSearchOptions(chatflow.workspaceId))
const { nodeOverrides, variableOverrides, apiOverrideStatus } = getAPIOverrideConfig(chatflow)
const upsertedResult = await buildFlow({
@@ -164,14 +182,18 @@ export const executeUpsert = async ({
sessionId,
chatflowid,
appDataSource,
usageCacheManager,
cachePool,
isUpsert,
stopNodeId,
overrideConfig,
apiOverrideStatus,
nodeOverrides,
availableVariables,
variableOverrides,
cachePool,
isUpsert,
stopNodeId
orgId,
workspaceId,
subscriptionId
})
// Save to DB
@@ -186,13 +208,17 @@ export const executeUpsert = async ({
await appDataSource.getRepository(UpsertHistory).save(upsertHistory)
}
await telemetry.sendTelemetry('vector_upserted', {
version: await getAppVersion(),
chatlowId: chatflowid,
type: isInternal ? ChatType.INTERNAL : ChatType.EXTERNAL,
flowGraph: getTelemetryFlowObj(nodes, edges),
stopNodeId
})
await telemetry.sendTelemetry(
'vector_upserted',
{
version: await getAppVersion(),
chatlowId: chatflowid,
type: isInternal ? ChatType.INTERNAL : ChatType.EXTERNAL,
flowGraph: getTelemetryFlowObj(nodes, edges),
stopNodeId
},
orgId
)
return upsertedResult['result'] ?? { result: 'Successfully Upserted' }
}
@@ -204,6 +230,7 @@ export const executeUpsert = async ({
*/
export const upsertVector = async (req: Request, isInternal: boolean = false) => {
const appServer = getRunningExpressApp()
try {
const chatflowid = req.params.id
@@ -228,6 +255,26 @@ export const upsertVector = async (req: Request, isInternal: boolean = false) =>
}
}
// This can be public API, so we can only get orgId from the chatflow
const chatflowWorkspaceId = chatflow.workspaceId
const workspace = await appServer.AppDataSource.getRepository(Workspace).findOneBy({
id: chatflowWorkspaceId
})
if (!workspace) {
throw new InternalFlowiseError(StatusCodes.NOT_FOUND, `Workspace ${chatflowWorkspaceId} not found`)
}
const workspaceId = workspace.id
const org = await appServer.AppDataSource.getRepository(Organization).findOneBy({
id: workspace.organizationId
})
if (!org) {
throw new InternalFlowiseError(StatusCodes.NOT_FOUND, `Organization ${workspace.organizationId} not found`)
}
const orgId = org.id
const subscriptionId = org.subscriptionId as string
const executeData: IExecuteFlowParams = {
componentNodes: appServer.nodesPool.componentNodes,
incomingInput,
@@ -237,17 +284,21 @@ export const upsertVector = async (req: Request, isInternal: boolean = false) =>
telemetry: appServer.telemetry,
cachePool: appServer.cachePool,
sseStreamer: appServer.sseStreamer,
usageCacheManager: appServer.usageCacheManager,
baseURL,
isInternal,
files,
isUpsert: true
isUpsert: true,
orgId,
workspaceId,
subscriptionId
}
if (process.env.MODE === MODE.QUEUE) {
const upsertQueue = appServer.queueManager.getQueue('upsert')
const job = await upsertQueue.addJob(omit(executeData, OMIT_QUEUE_JOB_DATA))
logger.debug(`[server]: Job added to queue: ${job.id}`)
logger.debug(`[server]: [${orgId}]: Job added to queue: ${job.id}`)
const queueEvents = upsertQueue.getQueueEvents()
const result = await job.waitUntilFinished(queueEvents)
packages/server/src/utils/validateKey.ts
+17
View File
@@ -34,6 +34,7 @@ export const validateAPIKey = async (req: Request) => {
if (!authorizationHeader) return false
const suppliedKey = authorizationHeader.split(`Bearer `).pop()
if (suppliedKey) {
const keys = await apikeyService.getAllApiKeys()
const apiSecret = keys.find((key: any) => key.apiKey === suppliedKey)?.apiSecret
@@ -43,3 +44,19 @@ export const validateAPIKey = async (req: Request) => {
}
return false
}
/**
* Get API Key WorkspaceID
* @param {Request} req
*/
export const getAPIKeyWorkspaceID = async (req: Request) => {
const authorizationHeader = (req.headers['Authorization'] as string) ?? (req.headers['authorization'] as string) ?? ''
if (!authorizationHeader) return false
const suppliedKey = authorizationHeader.split(`Bearer `).pop()
if (suppliedKey) {
const key = await apikeyService.getApiKey(suppliedKey)
return key?.workspaceId
}
return undefined
}