farcasclaudiu/Flowise
1
0
Fork 0
mirror of https://github.com/farcasclaudiu/Flowise.git synced 2026-06-28 15:00:57 +03:00
Code Issues Packages Projects Releases Wiki Activity

New Feature Pagination (#4704)

Browse Source 
* common pagination component

* Pagination for Doc Store Dashboard

* Pagination for Executions Dashboard

* Pagination Support for Tables

* lint fixes

* update view message dialog UI

* initial loading was ignoring the pagination counts

* 1) default page size change
2) ensure page limits are passed on load
3) co-pilot review comments (n+1 query)
4)

* 1) default page size change
2) ensure page limits are passed on load
3) co-pilot review comments (n+1 query)
4) refresh lists after insert/delete.

* Enhancement: Improve handling of empty responses in DocumentStore and API key services

- Added check for empty entities in DocumentStoreDTO.fromEntities to return an empty array.
- Updated condition in getAllDocumentStores to handle total count correctly, allowing for zero total.
- Refined logic in getAllApiKeys to check for empty keys and ensure correct API key retrieval.
- Adjusted UI components to safely handle potential undefined apiKeys array.

* Refresh API key list on pagination change

* Enhancement: Update pagination and filter handling across components
- Increased default items per page in AgentExecutions from 10 to 12.
- Improved JSON parsing for chat type and feedback type filters in ViewMessagesDialog.
- Enhanced execution filtering logic in AgentExecutions to ensure proper pagination and state management.
- Refactored filter section in AgentExecutions for better readability and functionality.
- Updated refresh logic in Agentflows to use the correct agentflow version.

* add workspaceId to removeAllChatMessages

* Refactor chat message retrieval logic for improved efficiency and maintainability

- Introduced a new `handleFeedbackQuery` function to streamline feedback-related queries.
- Enhanced pagination handling for session-based queries in `getMessagesWithFeedback`.
- Updated `ViewMessagesDialog` to sort messages in descending order by default.
- Simplified image rendering logic in `DocumentStoreTable` for better readability.

* - Update  `validateChatflowAPIKey` and `validateAPIKey` functions to get the correct keys array
- Enhanced error handling in the `sanitizeExecution` function to ensure safe access to nested properties

* Refactor API key validation logic for improved accuracy and error handling

- Consolidated API key validation in `validateAPIKey` to return detailed validation results.
- Updated `validateFlowAPIKey` to streamline flow API key validation.
- Introduced `getApiKeyById` function in the API key service for better key retrieval.
- Removed unused function `getAllChatSessionsFromChatflow` from the chat message API.

---------

Co-authored-by: Henry <hzj94@hotmail.com>
This commit is contained in:
Vinod Kiran
2025-07-10 20:29:24 +05:30
committed by GitHub
parent 6baec93860
commit bf05f25f7e
55 changed files with 2595 additions and 1560 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/server/src/Interface.DocumentStore.ts
+3
View File
@@ -290,6 +290,9 @@ export class DocumentStoreDTO {
}
static fromEntities(entities: DocumentStore[]): DocumentStoreDTO[] {
if (entities.length === 0) {
return []
}
return entities.map((entity) => this.fromEntity(entity))
}
packages/server/src/controllers/apikey/index.ts
+3 -1
View File
@@ -2,12 +2,14 @@ import { Request, Response, NextFunction } from 'express'
import { StatusCodes } from 'http-status-codes'
import { InternalFlowiseError } from '../../errors/internalFlowiseError'
import apikeyService from '../../services/apikey'
import { getPageAndLimitParams } from '../../utils/pagination'
// Get api keys
const getAllApiKeys = async (req: Request, res: Response, next: NextFunction) => {
try {
const autoCreateNewKey = true
const apiResponse = await apikeyService.getAllApiKeys(req.user?.activeWorkspaceId, autoCreateNewKey)
const { page, limit } = getPageAndLimitParams(req)
const apiResponse = await apikeyService.getAllApiKeys(req.user?.activeWorkspaceId, autoCreateNewKey, page, limit)
return res.json(apiResponse)
} catch (error) {
next(error)
packages/server/src/controllers/chat-messages/index.ts
+9 -2
View File
@@ -9,6 +9,7 @@ import { ChatMessage } from '../../database/entities/ChatMessage'
import { InternalFlowiseError } from '../../errors/internalFlowiseError'
import { StatusCodes } from 'http-status-codes'
import { utilGetChatMessage } from '../../utils/getChatMessage'
import { getPageAndLimitParams } from '../../utils/pagination'
const getFeedbackTypeFilters = (_feedbackTypeFilters: ChatMessageRatingType[]): ChatMessageRatingType[] | undefined => {
try {
@@ -71,6 +72,9 @@ const getAllChatMessages = async (req: Request, res: Response, next: NextFunctio
const startDate = req.query?.startDate as string | undefined
const endDate = req.query?.endDate as string | undefined
const feedback = req.query?.feedback as boolean | undefined
const { page, limit } = getPageAndLimitParams(req)
let feedbackTypeFilters = req.query?.feedbackType as ChatMessageRatingType[] | undefined
if (feedbackTypeFilters) {
feedbackTypeFilters = getFeedbackTypeFilters(feedbackTypeFilters)
@@ -93,7 +97,9 @@ const getAllChatMessages = async (req: Request, res: Response, next: NextFunctio
messageId,
feedback,
feedbackTypeFilters,
activeWorkspaceId
activeWorkspaceId,
page,
limit
)
return res.json(parseAPIResponse(apiResponse))
} catch (error) {
@@ -202,7 +208,8 @@ const removeAllChatMessages = async (req: Request, res: Response, next: NextFunc
startDate,
endDate,
feedback: isFeedback,
feedbackTypes: feedbackTypeFilters
feedbackTypes: feedbackTypeFilters,
activeWorkspaceId: workspaceId
})
const messageIds = messages.map((message) => message.id)
packages/server/src/controllers/chatflows/index.ts
+9 -1
View File
@@ -8,6 +8,7 @@ import chatflowsService from '../../services/chatflows'
import { getRunningExpressApp } from '../../utils/getRunningExpressApp'
import { checkUsageLimit } from '../../utils/quotaUsage'
import { RateLimiterManager } from '../../utils/rateLimit'
import { getPageAndLimitParams } from '../../utils/pagination'
const checkIfChatflowIsValidForStreaming = async (req: Request, res: Response, next: NextFunction) => {
try {
@@ -67,7 +68,14 @@ const deleteChatflow = async (req: Request, res: Response, next: NextFunction) =
const getAllChatflows = async (req: Request, res: Response, next: NextFunction) => {
try {
const apiResponse = await chatflowsService.getAllChatflows(req.query?.type as ChatflowType, req.user?.activeWorkspaceId)
const { page, limit } = getPageAndLimitParams(req)
const apiResponse = await chatflowsService.getAllChatflows(
req.query?.type as ChatflowType,
req.user?.activeWorkspaceId,
page,
limit
)
return res.json(apiResponse)
} catch (error) {
next(error)
packages/server/src/controllers/dataset/index.ts
+5 -2
View File
@@ -2,10 +2,12 @@ import { Request, Response, NextFunction } from 'express'
import { InternalFlowiseError } from '../../errors/internalFlowiseError'
import datasetService from '../../services/dataset'
import { StatusCodes } from 'http-status-codes'
import { getPageAndLimitParams } from '../../utils/pagination'
const getAllDatasets = async (req: Request, res: Response, next: NextFunction) => {
try {
const apiResponse = await datasetService.getAllDatasets(req.user?.activeWorkspaceId)
const { page, limit } = getPageAndLimitParams(req)
const apiResponse = await datasetService.getAllDatasets(req.user?.activeWorkspaceId, page, limit)
return res.json(apiResponse)
} catch (error) {
next(error)
@@ -17,7 +19,8 @@ const getDataset = async (req: Request, res: Response, next: NextFunction) => {
if (typeof req.params === 'undefined' || !req.params.id) {
throw new InternalFlowiseError(StatusCodes.PRECONDITION_FAILED, `Error: datasetService.getDataset - id not provided!`)
}
const apiResponse = await datasetService.getDataset(req.params.id)
const { page, limit } = getPageAndLimitParams(req)
const apiResponse = await datasetService.getDataset(req.params.id, page, limit)
return res.json(apiResponse)
} catch (error) {
next(error)
packages/server/src/controllers/documentstore/index.ts
+12 -2
View File
@@ -6,6 +6,7 @@ import { InternalFlowiseError } from '../../errors/internalFlowiseError'
import { DocumentStoreDTO } from '../../Interface'
import { getRunningExpressApp } from '../../utils/getRunningExpressApp'
import { FLOWISE_COUNTER_STATUS, FLOWISE_METRIC_COUNTERS } from '../../Interface.Metrics'
import { getPageAndLimitParams } from '../../utils/pagination'
const createDocumentStore = async (req: Request, res: Response, next: NextFunction) => {
try {
@@ -37,8 +38,17 @@ const createDocumentStore = async (req: Request, res: Response, next: NextFuncti
const getAllDocumentStores = async (req: Request, res: Response, next: NextFunction) => {
try {
const apiResponse = await documentStoreService.getAllDocumentStores(req.user?.activeWorkspaceId)
return res.json(DocumentStoreDTO.fromEntities(apiResponse))
const { page, limit } = getPageAndLimitParams(req)
const apiResponse: any = await documentStoreService.getAllDocumentStores(req.user?.activeWorkspaceId, page, limit)
if (apiResponse?.total >= 0) {
return res.json({
total: apiResponse.total,
data: DocumentStoreDTO.fromEntities(apiResponse.data)
})
} else {
return res.json(DocumentStoreDTO.fromEntities(apiResponse))
}
} catch (error) {
next(error)
}
packages/server/src/controllers/evaluations/index.ts
+3 -1
View File
@@ -2,6 +2,7 @@ import { Request, Response, NextFunction } from 'express'
import { InternalFlowiseError } from '../../errors/internalFlowiseError'
import { StatusCodes } from 'http-status-codes'
import evaluationsService from '../../services/evaluations'
import { getPageAndLimitParams } from '../../utils/pagination'
const createEvaluation = async (req: Request, res: Response, next: NextFunction) => {
try {
@@ -81,7 +82,8 @@ const deleteEvaluation = async (req: Request, res: Response, next: NextFunction)
const getAllEvaluations = async (req: Request, res: Response, next: NextFunction) => {
try {
const apiResponse = await evaluationsService.getAllEvaluations(req.user?.activeWorkspaceId)
const { page, limit } = getPageAndLimitParams(req)
const apiResponse = await evaluationsService.getAllEvaluations(req.user?.activeWorkspaceId, page, limit)
return res.json(apiResponse)
} catch (error) {
next(error)
packages/server/src/controllers/evaluators/index.ts
+3 -1
View File
@@ -2,10 +2,12 @@ import { Request, Response, NextFunction } from 'express'
import { InternalFlowiseError } from '../../errors/internalFlowiseError'
import { StatusCodes } from 'http-status-codes'
import evaluatorService from '../../services/evaluator'
import { getPageAndLimitParams } from '../../utils/pagination'
const getAllEvaluators = async (req: Request, res: Response, next: NextFunction) => {
try {
const apiResponse = await evaluatorService.getAllEvaluators(req.user?.activeWorkspaceId)
const { page, limit } = getPageAndLimitParams(req)
const apiResponse = await evaluatorService.getAllEvaluators(req.user?.activeWorkspaceId, page, limit)
return res.json(apiResponse)
} catch (error) {
next(error)
packages/server/src/controllers/stats/index.ts
+10 -1
View File
@@ -45,7 +45,16 @@ const getChatflowStats = async (req: Request, res: Response, next: NextFunction)
return res.status(500).send(e)
}
}
const apiResponse = await statsService.getChatflowStats(chatflowid, chatTypes, startDate, endDate, '', true, feedbackTypeFilters)
const apiResponse = await statsService.getChatflowStats(
chatflowid,
chatTypes,
startDate,
endDate,
'',
true,
feedbackTypeFilters,
req.user?.activeWorkspaceId
)
return res.json(apiResponse)
} catch (error) {
next(error)
packages/server/src/controllers/tools/index.ts
+3 -1
View File
@@ -2,6 +2,7 @@ import { NextFunction, Request, Response } from 'express'
import toolsService from '../../services/tools'
import { InternalFlowiseError } from '../../errors/internalFlowiseError'
import { StatusCodes } from 'http-status-codes'
import { getPageAndLimitParams } from '../../utils/pagination'
const createTool = async (req: Request, res: Response, next: NextFunction) => {
try {
@@ -40,7 +41,8 @@ const deleteTool = async (req: Request, res: Response, next: NextFunction) => {
const getAllTools = async (req: Request, res: Response, next: NextFunction) => {
try {
const apiResponse = await toolsService.getAllTools(req.user?.activeWorkspaceId)
const { page, limit } = getPageAndLimitParams(req)
const apiResponse = await toolsService.getAllTools(req.user?.activeWorkspaceId, page, limit)
return res.json(apiResponse)
} catch (error) {
next(error)
packages/server/src/controllers/variables/index.ts
+3 -1
View File
@@ -3,6 +3,7 @@ import variablesService from '../../services/variables'
import { Variable } from '../../database/entities/Variable'
import { InternalFlowiseError } from '../../errors/internalFlowiseError'
import { StatusCodes } from 'http-status-codes'
import { getPageAndLimitParams } from '../../utils/pagination'
const createVariable = async (req: Request, res: Response, next: NextFunction) => {
try {
@@ -45,7 +46,8 @@ const deleteVariable = async (req: Request, res: Response, next: NextFunction) =
const getAllVariables = async (req: Request, res: Response, next: NextFunction) => {
try {
const apiResponse = await variablesService.getAllVariables(req.user?.activeWorkspaceId)
const { page, limit } = getPageAndLimitParams(req)
const apiResponse = await variablesService.getAllVariables(req.user?.activeWorkspaceId, page, limit)
return res.json(apiResponse)
} catch (error) {
next(error)
packages/server/src/index.ts
+45 -48
View File
@@ -21,7 +21,7 @@ import { WHITELIST_URLS } from './utils/constants'
import { initializeJwtCookieMiddleware, verifyToken } from './enterprise/middleware/passport'
import { IdentityManager } from './IdentityManager'
import { SSEStreamer } from './utils/SSEStreamer'
import { getAPIKeyWorkspaceID, validateAPIKey } from './utils/validateKey'
import { validateAPIKey } from './utils/validateKey'
import { LoggedInUser } from './enterprise/Interface.Enterprise'
import { IMetricsProvider } from './Interface.Metrics'
import { Prometheus } from './metrics/Prometheus'
@@ -217,58 +217,55 @@ export class App {
return res.status(401).json({ error: 'Unauthorized Access' })
}
}
const isKeyValidated = await validateAPIKey(req)
if (!isKeyValidated) {
const { isValid, workspaceId: apiKeyWorkSpaceId } = await validateAPIKey(req)
if (!isValid) {
return res.status(401).json({ error: 'Unauthorized Access' })
}
const apiKeyWorkSpaceId = await getAPIKeyWorkspaceID(req)
if (apiKeyWorkSpaceId) {
// Find workspace
const workspace = await this.AppDataSource.getRepository(Workspace).findOne({
where: { id: apiKeyWorkSpaceId }
})
if (!workspace) {
return res.status(401).json({ error: 'Unauthorized Access' })
}
// Find owner role
const ownerRole = await this.AppDataSource.getRepository(Role).findOne({
where: { name: GeneralRole.OWNER, organizationId: IsNull() }
})
if (!ownerRole) {
return res.status(401).json({ error: 'Unauthorized Access' })
}
// Find organization
const activeOrganizationId = workspace.organizationId as string
const org = await this.AppDataSource.getRepository(Organization).findOne({
where: { id: activeOrganizationId }
})
if (!org) {
return res.status(401).json({ error: 'Unauthorized Access' })
}
const subscriptionId = org.subscriptionId as string
const customerId = org.customerId as string
const features = await this.identityManager.getFeaturesByPlan(subscriptionId)
const productId = await this.identityManager.getProductIdFromSubscription(subscriptionId)
// @ts-ignore
req.user = {
permissions: [...JSON.parse(ownerRole.permissions)],
features,
activeOrganizationId: activeOrganizationId,
activeOrganizationSubscriptionId: subscriptionId,
activeOrganizationCustomerId: customerId,
activeOrganizationProductId: productId,
isOrganizationAdmin: true,
activeWorkspaceId: apiKeyWorkSpaceId,
activeWorkspace: workspace.name,
isApiKeyValidated: true
}
next()
} else {
// Find workspace
const workspace = await this.AppDataSource.getRepository(Workspace).findOne({
where: { id: apiKeyWorkSpaceId }
})
if (!workspace) {
return res.status(401).json({ error: 'Unauthorized Access' })
}
// Find owner role
const ownerRole = await this.AppDataSource.getRepository(Role).findOne({
where: { name: GeneralRole.OWNER, organizationId: IsNull() }
})
if (!ownerRole) {
return res.status(401).json({ error: 'Unauthorized Access' })
}
// Find organization
const activeOrganizationId = workspace.organizationId as string
const org = await this.AppDataSource.getRepository(Organization).findOne({
where: { id: activeOrganizationId }
})
if (!org) {
return res.status(401).json({ error: 'Unauthorized Access' })
}
const subscriptionId = org.subscriptionId as string
const customerId = org.customerId as string
const features = await this.identityManager.getFeaturesByPlan(subscriptionId)
const productId = await this.identityManager.getProductIdFromSubscription(subscriptionId)
// @ts-ignore
req.user = {
permissions: [...JSON.parse(ownerRole.permissions)],
features,
activeOrganizationId: activeOrganizationId,
activeOrganizationSubscriptionId: subscriptionId,
activeOrganizationCustomerId: customerId,
activeOrganizationProductId: productId,
isOrganizationAdmin: true,
activeWorkspaceId: apiKeyWorkSpaceId!,
activeWorkspace: workspace.name,
isApiKeyValidated: true
}
next()
}
} else {
return res.status(401).json({ error: 'Unauthorized Access' })
packages/server/src/services/apikey/index.ts
+36 -8
View File
@@ -9,19 +9,31 @@ import { Not, IsNull } from 'typeorm'
import { getWorkspaceSearchOptions } from '../../enterprise/utils/ControllerServiceUtils'
import { v4 as uuidv4 } from 'uuid'
const getAllApiKeysFromDB = async (workspaceId?: string) => {
const getAllApiKeysFromDB = async (workspaceId?: string, page: number = -1, limit: number = -1) => {
const appServer = getRunningExpressApp()
const keys = await appServer.AppDataSource.getRepository(ApiKey).findBy(getWorkspaceSearchOptions(workspaceId))
const keysWithChatflows = await addChatflowsCount(keys)
return keysWithChatflows
const queryBuilder = appServer.AppDataSource.getRepository(ApiKey).createQueryBuilder('api_key').orderBy('api_key.updatedDate', 'DESC')
if (page > 0 && limit > 0) {
queryBuilder.skip((page - 1) * limit)
queryBuilder.take(limit)
}
if (workspaceId) queryBuilder.andWhere('api_key.workspaceId = :workspaceId', { workspaceId })
const [data, total] = await queryBuilder.getManyAndCount()
const keysWithChatflows = await addChatflowsCount(data)
if (page > 0 && limit > 0) {
return { total, data: keysWithChatflows }
} else {
return keysWithChatflows
}
}
const getAllApiKeys = async (workspaceId?: string, autoCreateNewKey?: boolean) => {
const getAllApiKeys = async (workspaceId?: string, autoCreateNewKey?: boolean, page: number = -1, limit: number = -1) => {
try {
let keys = await getAllApiKeysFromDB(workspaceId)
if (keys.length === 0 && autoCreateNewKey) {
let keys = await getAllApiKeysFromDB(workspaceId, page, limit)
const isEmpty = keys?.total === 0 || (Array.isArray(keys) && keys?.length === 0)
if (isEmpty && autoCreateNewKey) {
await createApiKey('DefaultKey', workspaceId)
keys = await getAllApiKeysFromDB(workspaceId)
keys = await getAllApiKeysFromDB(workspaceId, page, limit)
}
return keys
} catch (error) {
@@ -44,6 +56,21 @@ const getApiKey = async (apiKey: string) => {
}
}
const getApiKeyById = async (apiKeyId: string) => {
try {
const appServer = getRunningExpressApp()
const currentKey = await appServer.AppDataSource.getRepository(ApiKey).findOneBy({
id: apiKeyId
})
if (!currentKey) {
return undefined
}
return currentKey
} catch (error) {
throw new InternalFlowiseError(StatusCodes.INTERNAL_SERVER_ERROR, `Error: apikeyService.getApiKeyById - ${getErrorMessage(error)}`)
}
}
const createApiKey = async (keyName: string, workspaceId?: string) => {
try {
const apiKey = generateAPIKey()
@@ -231,5 +258,6 @@ export default {
updateApiKey,
verifyApiKey,
getApiKey,
getApiKeyById,
importKeys
}
packages/server/src/services/chat-messages/index.ts
+6 -2
View File
@@ -39,7 +39,9 @@ const getAllChatMessages = async (
messageId?: string,
feedback?: boolean,
feedbackTypes?: ChatMessageRatingType[],
activeWorkspaceId?: string
activeWorkspaceId?: string,
page?: number,
pageSize?: number
): Promise<ChatMessage[]> => {
try {
const dbResponse = await utilGetChatMessage({
@@ -54,7 +56,9 @@ const getAllChatMessages = async (
messageId,
feedback,
feedbackTypes,
activeWorkspaceId
activeWorkspaceId,
page,
pageSize
})
return dbResponse
} catch (error) {
packages/server/src/services/chatflows/index.ts
+22 -7
View File
@@ -127,21 +127,36 @@ const deleteChatflow = async (chatflowId: string, orgId: string, workspaceId: st
}
}
const getAllChatflows = async (type?: ChatflowType, workspaceId?: string): Promise<ChatFlow[]> => {
const getAllChatflows = async (type?: ChatflowType, workspaceId?: string, page: number = -1, limit: number = -1) => {
try {
const appServer = getRunningExpressApp()
const dbResponse = await appServer.AppDataSource.getRepository(ChatFlow).findBy(getWorkspaceSearchOptions(workspaceId))
const queryBuilder = appServer.AppDataSource.getRepository(ChatFlow)
.createQueryBuilder('chat_flow')
.orderBy('chat_flow.updatedDate', 'DESC')
if (page > 0 && limit > 0) {
queryBuilder.skip((page - 1) * limit)
queryBuilder.take(limit)
}
if (type === 'MULTIAGENT') {
return dbResponse.filter((chatflow) => chatflow.type === 'MULTIAGENT')
queryBuilder.andWhere('chat_flow.type = :type', { type: 'MULTIAGENT' })
} else if (type === 'AGENTFLOW') {
return dbResponse.filter((chatflow) => chatflow.type === 'AGENTFLOW')
queryBuilder.andWhere('chat_flow.type = :type', { type: 'AGENTFLOW' })
} else if (type === 'ASSISTANT') {
return dbResponse.filter((chatflow) => chatflow.type === 'ASSISTANT')
queryBuilder.andWhere('chat_flow.type = :type', { type: 'ASSISTANT' })
} else if (type === 'CHATFLOW') {
// fetch all chatflows that are not agentflow
return dbResponse.filter((chatflow) => chatflow.type === 'CHATFLOW' || !chatflow.type)
queryBuilder.andWhere('chat_flow.type = :type', { type: 'CHATFLOW' })
}
if (workspaceId) queryBuilder.andWhere('chat_flow.workspaceId = :workspaceId', { workspaceId })
const [data, total] = await queryBuilder.getManyAndCount()
if (page > 0 && limit > 0) {
return { data, total }
} else {
return data
}
return dbResponse
} catch (error) {
throw new InternalFlowiseError(
StatusCodes.INTERNAL_SERVER_ERROR,
packages/server/src/services/dataset/index.ts
+37 -17
View File
@@ -8,22 +8,33 @@ import { Readable } from 'stream'
import { In } from 'typeorm'
import csv from 'csv-parser'
import { getWorkspaceSearchOptions } from '../../enterprise/utils/ControllerServiceUtils'
const getAllDatasets = async (workspaceId?: string) => {
const getAllDatasets = async (workspaceId?: string, page: number = -1, limit: number = -1) => {
try {
const appServer = getRunningExpressApp()
const queryBuilder = appServer.AppDataSource.getRepository(Dataset).createQueryBuilder('ds').orderBy('ds.updatedDate', 'DESC')
if (page > 0 && limit > 0) {
queryBuilder.skip((page - 1) * limit)
queryBuilder.take(limit)
}
if (workspaceId) queryBuilder.andWhere('ds.workspaceId = :workspaceId', { workspaceId })
const [data, total] = await queryBuilder.getManyAndCount()
const returnObj: Dataset[] = []
const datasets = await appServer.AppDataSource.getRepository(Dataset).findBy(getWorkspaceSearchOptions(workspaceId))
// TODO: This is a hack to get the row count for each dataset. Need to find a better way to do this
for (const dataset of datasets) {
for (const dataset of data) {
;(dataset as any).rowCount = await appServer.AppDataSource.getRepository(DatasetRow).count({
where: { datasetId: dataset.id }
})
returnObj.push(dataset)
}
return returnObj
if (page > 0 && limit > 0) {
return { total, data: returnObj }
} else {
return returnObj
}
} catch (error) {
throw new InternalFlowiseError(
StatusCodes.INTERNAL_SERVER_ERROR,
@@ -32,36 +43,45 @@ const getAllDatasets = async (workspaceId?: string) => {
}
}
const getDataset = async (id: string) => {
const getDataset = async (id: string, page: number = -1, limit: number = -1) => {
try {
const appServer = getRunningExpressApp()
const dataset = await appServer.AppDataSource.getRepository(Dataset).findOneBy({
id: id
})
let items = await appServer.AppDataSource.getRepository(DatasetRow).find({
where: { datasetId: id },
order: { sequenceNo: 'asc' }
})
const queryBuilder = appServer.AppDataSource.getRepository(DatasetRow).createQueryBuilder('dsr').orderBy('dsr.sequenceNo', 'ASC')
queryBuilder.andWhere('dsr.datasetId = :datasetId', { datasetId: id })
if (page > 0 && limit > 0) {
queryBuilder.skip((page - 1) * limit)
queryBuilder.take(limit)
}
let [data, total] = await queryBuilder.getManyAndCount()
// special case for sequence numbers == -1 (this happens when the update script is run and all rows are set to -1)
// check if there are any sequence numbers == -1, if so set them to the max sequence number + 1
const missingSequenceNumbers = items.filter((item) => item.sequenceNo === -1)
const missingSequenceNumbers = data.filter((item) => item.sequenceNo === -1)
if (missingSequenceNumbers.length > 0) {
const maxSequenceNumber = items.reduce((prev, current) => (prev.sequenceNo > current.sequenceNo ? prev : current))
const maxSequenceNumber = data.reduce((prev, current) => (prev.sequenceNo > current.sequenceNo ? prev : current))
let sequenceNo = maxSequenceNumber.sequenceNo + 1
for (const zeroSequenceNumber of missingSequenceNumbers) {
zeroSequenceNumber.sequenceNo = sequenceNo++
}
await appServer.AppDataSource.getRepository(DatasetRow).save(missingSequenceNumbers)
// now get the items again
items = await appServer.AppDataSource.getRepository(DatasetRow).find({
where: { datasetId: id },
order: { sequenceNo: 'asc' }
})
const queryBuilder2 = appServer.AppDataSource.getRepository(DatasetRow)
.createQueryBuilder('dsr')
.orderBy('dsr.sequenceNo', 'ASC')
queryBuilder2.andWhere('dsr.datasetId = :datasetId', { datasetId: id })
if (page > 0 && limit > 0) {
queryBuilder2.skip((page - 1) * limit)
queryBuilder2.take(limit)
}
;[data, total] = await queryBuilder2.getManyAndCount()
}
return {
...dataset,
rows: items
rows: data,
total
}
} catch (error) {
throw new InternalFlowiseError(StatusCodes.INTERNAL_SERVER_ERROR, `Error: datasetService.getDataset - ${getErrorMessage(error)}`)
packages/server/src/services/documentstore/index.ts
+18 -3
View File
@@ -77,11 +77,26 @@ const createDocumentStore = async (newDocumentStore: DocumentStore, orgId: strin
}
}
const getAllDocumentStores = async (workspaceId?: string) => {
const getAllDocumentStores = async (workspaceId?: string, page: number = -1, limit: number = -1) => {
try {
const appServer = getRunningExpressApp()
const entities = await appServer.AppDataSource.getRepository(DocumentStore).findBy(getWorkspaceSearchOptions(workspaceId))
return entities
const queryBuilder = appServer.AppDataSource.getRepository(DocumentStore)
.createQueryBuilder('doc_store')
.orderBy('doc_store.updatedDate', 'DESC')
if (page > 0 && limit > 0) {
queryBuilder.skip((page - 1) * limit)
queryBuilder.take(limit)
}
if (workspaceId) queryBuilder.andWhere('doc_store.workspaceId = :workspaceId', { workspaceId })
const [data, total] = await queryBuilder.getManyAndCount()
if (page > 0 && limit > 0) {
return { data, total }
} else {
return data
}
} catch (error) {
throw new InternalFlowiseError(
StatusCodes.INTERNAL_SERVER_ERROR,
packages/server/src/services/evaluations/index.ts
+65 -27
View File
@@ -338,42 +338,80 @@ const createEvaluation = async (body: ICommonObject, baseURL: string, orgId: str
}
}
const getAllEvaluations = async (workspaceId?: string) => {
const getAllEvaluations = async (workspaceId?: string, page: number = -1, limit: number = -1) => {
try {
const appServer = getRunningExpressApp()
const findAndOrderBy: any = {
where: getWorkspaceSearchOptions(workspaceId),
order: {
runDate: 'DESC'
}
}
const evaluations = await appServer.AppDataSource.getRepository(Evaluation).find(findAndOrderBy)
// First, get the count of distinct evaluation names for the total
// needed as the The getCount() method in TypeORM doesn't respect the GROUP BY clause and will return the total count of records
const countQuery = appServer.AppDataSource.getRepository(Evaluation)
.createQueryBuilder('ev')
.select('COUNT(DISTINCT(ev.name))', 'count')
.where('ev.workspaceId = :workspaceId', { workspaceId: workspaceId })
const totalResult = await countQuery.getRawOne()
const total = totalResult ? parseInt(totalResult.count) : 0
// Then get the distinct evaluation names with their counts and latest run date
const namesQueryBuilder = appServer.AppDataSource.getRepository(Evaluation)
.createQueryBuilder('ev')
.select('DISTINCT(ev.name)', 'name')
.addSelect('COUNT(ev.name)', 'count')
.addSelect('MAX(ev.runDate)', 'latestRunDate')
.andWhere('ev.workspaceId = :workspaceId', { workspaceId: workspaceId })
.groupBy('ev.name')
.orderBy('max(ev.runDate)', 'DESC') // Order by the latest run date
if (page > 0 && limit > 0) {
namesQueryBuilder.skip((page - 1) * limit)
namesQueryBuilder.take(limit)
}
const evaluationNames = await namesQueryBuilder.getRawMany()
// Get all evaluations for all names at once in a single query
const returnResults: IEvaluationResult[] = []
// mark the first evaluation with a unique name as the latestEval and then reset the version number
for (let i = 0; i < evaluations.length; i++) {
const evaluation = evaluations[i] as IEvaluationResult
returnResults.push(evaluation)
// find the first index with this name in the evaluations array
// as it is sorted desc, make the first evaluation with this name as the latestEval
const currentIndex = evaluations.indexOf(evaluation)
if (evaluations.findIndex((e) => e.name === evaluation.name) === currentIndex) {
returnResults[i].latestEval = true
}
}
for (let i = 0; i < returnResults.length; i++) {
const evaluation = returnResults[i]
if (evaluation.latestEval) {
const versions = returnResults.filter((e) => e.name === evaluation.name)
let descVersion = versions.length
for (let j = 0; j < versions.length; j++) {
versions[j].version = descVersion--
if (evaluationNames.length > 0) {
const names = evaluationNames.map((item) => item.name)
// Fetch all evaluations for these names in a single query
const allEvaluations = await appServer.AppDataSource.getRepository(Evaluation)
.createQueryBuilder('ev')
.where('ev.name IN (:...names)', { names })
.andWhere('ev.workspaceId = :workspaceId', { workspaceId })
.orderBy('ev.name', 'ASC')
.addOrderBy('ev.runDate', 'DESC')
.getMany()
// Process the results by name
const evaluationsByName = new Map<string, Evaluation[]>()
// Group evaluations by name
for (const evaluation of allEvaluations) {
if (!evaluationsByName.has(evaluation.name)) {
evaluationsByName.set(evaluation.name, [])
}
evaluationsByName.get(evaluation.name)!.push(evaluation)
}
// Process each name's evaluations
for (const item of evaluationNames) {
const evaluationsForName = evaluationsByName.get(item.name) || []
for (let i = 0; i < evaluationsForName.length; i++) {
const evaluation = evaluationsForName[i] as IEvaluationResult
evaluation.latestEval = i === 0
evaluation.version = parseInt(item.count) - i
returnResults.push(evaluation)
}
}
}
return returnResults
if (page > 0 && limit > 0) {
return {
total: total,
data: returnResults
}
} else {
return returnResults
}
} catch (error) {
throw new InternalFlowiseError(
StatusCodes.INTERNAL_SERVER_ERROR,
packages/server/src/services/evaluator/index.ts
+16 -4
View File
@@ -4,13 +4,25 @@ import { InternalFlowiseError } from '../../errors/internalFlowiseError'
import { getErrorMessage } from '../../errors/utils'
import { Evaluator } from '../../database/entities/Evaluator'
import { EvaluatorDTO } from '../../Interface.Evaluation'
import { getWorkspaceSearchOptions } from '../../enterprise/utils/ControllerServiceUtils'
const getAllEvaluators = async (workspaceId?: string) => {
const getAllEvaluators = async (workspaceId?: string, page: number = -1, limit: number = -1) => {
try {
const appServer = getRunningExpressApp()
const results: Evaluator[] = await appServer.AppDataSource.getRepository(Evaluator).findBy(getWorkspaceSearchOptions(workspaceId))
return EvaluatorDTO.fromEntities(results)
const queryBuilder = appServer.AppDataSource.getRepository(Evaluator).createQueryBuilder('ev').orderBy('ev.updatedDate', 'DESC')
if (workspaceId) queryBuilder.andWhere('ev.workspaceId = :workspaceId', { workspaceId })
if (page > 0 && limit > 0) {
queryBuilder.skip((page - 1) * limit)
queryBuilder.take(limit)
}
const [data, total] = await queryBuilder.getManyAndCount()
if (page > 0 && limit > 0) {
return {
total,
data: EvaluatorDTO.fromEntities(data)
}
} else {
return EvaluatorDTO.fromEntities(data)
}
} catch (error) {
throw new InternalFlowiseError(
StatusCodes.INTERNAL_SERVER_ERROR,
packages/server/src/services/executions/index.ts
+1 -1
View File
@@ -65,7 +65,7 @@ const getPublicExecutionById = async (executionId: string): Promise<Execution |
const getAllExecutions = async (filters: ExecutionFilters = {}): Promise<{ data: Execution[]; total: number }> => {
try {
const appServer = getRunningExpressApp()
const { id, agentflowId, sessionId, state, startDate, endDate, page = 1, limit = 10, workspaceId } = filters
const { id, agentflowId, sessionId, state, startDate, endDate, page = 1, limit = 12, workspaceId } = filters
// Handle UUID fields properly using raw parameters to avoid type conversion issues
// This uses the query builder instead of direct objects for compatibility with UUID fields
packages/server/src/services/export-import/index.ts
+21 -8
View File
@@ -90,16 +90,20 @@ const convertExportInput = (body: any): ExportInput => {
const FileDefaultName = 'ExportData.json'
const exportData = async (exportInput: ExportInput, activeWorkspaceId?: string): Promise<{ FileDefaultName: string } & ExportData> => {
try {
let AgentFlow: ChatFlow[] =
let AgentFlow: ChatFlow[] | { data: ChatFlow[]; total: number } =
exportInput.agentflow === true ? await chatflowService.getAllChatflows('MULTIAGENT', activeWorkspaceId) : []
AgentFlow = 'data' in AgentFlow ? AgentFlow.data : AgentFlow
let AgentFlowV2: ChatFlow[] =
let AgentFlowV2: ChatFlow[] | { data: ChatFlow[]; total: number } =
exportInput.agentflowv2 === true ? await chatflowService.getAllChatflows('AGENTFLOW', activeWorkspaceId) : []
AgentFlowV2 = 'data' in AgentFlowV2 ? AgentFlowV2.data : AgentFlowV2
let AssistantCustom: Assistant[] =
exportInput.assistantCustom === true ? await assistantService.getAllAssistants('CUSTOM', activeWorkspaceId) : []
let AssistantFlow: ChatFlow[] =
let AssistantFlow: ChatFlow[] | { data: ChatFlow[]; total: number } =
exportInput.assistantCustom === true ? await chatflowService.getAllChatflows('ASSISTANT', activeWorkspaceId) : []
AssistantFlow = 'data' in AssistantFlow ? AssistantFlow.data : AssistantFlow
let AssistantOpenAI: Assistant[] =
exportInput.assistantOpenAI === true ? await assistantService.getAllAssistants('OPENAI', activeWorkspaceId) : []
@@ -107,12 +111,15 @@ const exportData = async (exportInput: ExportInput, activeWorkspaceId?: string):
let AssistantAzure: Assistant[] =
exportInput.assistantAzure === true ? await assistantService.getAllAssistants('AZURE', activeWorkspaceId) : []
let ChatFlow: ChatFlow[] = exportInput.chatflow === true ? await chatflowService.getAllChatflows('CHATFLOW', activeWorkspaceId) : []
let ChatFlow: ChatFlow[] | { data: ChatFlow[]; total: number } =
exportInput.chatflow === true ? await chatflowService.getAllChatflows('CHATFLOW', activeWorkspaceId) : []
ChatFlow = 'data' in ChatFlow ? ChatFlow.data : ChatFlow
const allChatflow: ChatFlow[] =
let allChatflow: ChatFlow[] | { data: ChatFlow[]; total: number } =
exportInput.chat_message === true || exportInput.chat_feedback === true
? await chatflowService.getAllChatflows(undefined, activeWorkspaceId)
: []
allChatflow = 'data' in allChatflow ? allChatflow.data : allChatflow
const chatflowIds = allChatflow.map((chatflow) => chatflow.id)
let ChatMessage: ChatMessage[] =
@@ -124,8 +131,10 @@ const exportData = async (exportInput: ExportInput, activeWorkspaceId?: string):
let CustomTemplate: CustomTemplate[] =
exportInput.custom_template === true ? await marketplacesService.getAllCustomTemplates(activeWorkspaceId) : []
let DocumentStore: DocumentStore[] =
let DocumentStore: DocumentStore[] | { data: DocumentStore[]; total: number } =
exportInput.document_store === true ? await documenStoreService.getAllDocumentStores(activeWorkspaceId) : []
DocumentStore = 'data' in DocumentStore ? DocumentStore.data : DocumentStore
const documentStoreIds = DocumentStore.map((documentStore) => documentStore.id)
let DocumentStoreFileChunk: DocumentStoreFileChunk[] =
@@ -137,9 +146,13 @@ const exportData = async (exportInput: ExportInput, activeWorkspaceId?: string):
const { data: totalExecutions } = exportInput.execution === true ? await executionService.getAllExecutions(filters) : { data: [] }
let Execution: Execution[] = exportInput.execution === true ? totalExecutions : []
let Tool: Tool[] = exportInput.tool === true ? await toolsService.getAllTools(activeWorkspaceId) : []
let Tool: Tool[] | { data: Tool[]; total: number } =
exportInput.tool === true ? await toolsService.getAllTools(activeWorkspaceId) : []
Tool = 'data' in Tool ? Tool.data : Tool
let Variable: Variable[] = exportInput.variable === true ? await variableService.getAllVariables(activeWorkspaceId) : []
let Variable: Variable[] | { data: Variable[]; total: number } =
exportInput.variable === true ? await variableService.getAllVariables(activeWorkspaceId) : []
Variable = 'data' in Variable ? Variable.data : Variable
return {
FileDefaultName,
packages/server/src/services/stats/index.ts
+9 -3
View File
@@ -14,7 +14,8 @@ const getChatflowStats = async (
endDate?: string,
messageId?: string,
feedback?: boolean,
feedbackTypes?: ChatMessageRatingType[]
feedbackTypes?: ChatMessageRatingType[],
activeWorkspaceId?: string
): Promise<any> => {
try {
const chatmessages = (await utilGetChatMessage({
@@ -24,15 +25,20 @@ const getChatflowStats = async (
endDate,
messageId,
feedback,
feedbackTypes
feedbackTypes,
activeWorkspaceId
})) as Array<ChatMessage & { feedback?: ChatMessageFeedback }>
const totalMessages = chatmessages.length
const totalFeedback = chatmessages.filter((message) => message?.feedback).length
const positiveFeedback = chatmessages.filter((message) => message?.feedback?.rating === 'THUMBS_UP').length
// count the number of unique sessions in the chatmessages - count unique sessionId
const uniqueSessions = new Set(chatmessages.map((message) => message.sessionId))
const totalSessions = uniqueSessions.size
const dbResponse = {
totalMessages,
totalFeedback,
positiveFeedback
positiveFeedback,
totalSessions
}
return dbResponse
packages/server/src/services/tools/index.ts
+15 -4
View File
@@ -3,7 +3,6 @@ import { Tool } from '../../database/entities/Tool'
import { getAppVersion } from '../../utils'
import { InternalFlowiseError } from '../../errors/internalFlowiseError'
import { getErrorMessage } from '../../errors/utils'
import { getWorkspaceSearchOptions } from '../../enterprise/utils/ControllerServiceUtils'
import { getRunningExpressApp } from '../../utils/getRunningExpressApp'
import { FLOWISE_METRIC_COUNTERS, FLOWISE_COUNTER_STATUS } from '../../Interface.Metrics'
import { QueryRunner } from 'typeorm'
@@ -44,11 +43,23 @@ const deleteTool = async (toolId: string): Promise<any> => {
}
}
const getAllTools = async (workspaceId?: string): Promise<Tool[]> => {
const getAllTools = async (workspaceId?: string, page: number = -1, limit: number = -1) => {
try {
const appServer = getRunningExpressApp()
const dbResponse = await appServer.AppDataSource.getRepository(Tool).findBy(getWorkspaceSearchOptions(workspaceId))
return dbResponse
const queryBuilder = appServer.AppDataSource.getRepository(Tool).createQueryBuilder('tool').orderBy('tool.updatedDate', 'DESC')
if (page > 0 && limit > 0) {
queryBuilder.skip((page - 1) * limit)
queryBuilder.take(limit)
}
if (workspaceId) queryBuilder.andWhere('tool.workspaceId = :workspaceId', { workspaceId })
const [data, total] = await queryBuilder.getManyAndCount()
if (page > 0 && limit > 0) {
return { data, total }
} else {
return data
}
} catch (error) {
throw new InternalFlowiseError(StatusCodes.INTERNAL_SERVER_ERROR, `Error: toolsService.getAllTools - ${getErrorMessage(error)}`)
}
packages/server/src/services/variables/index.ts
+18 -4
View File
@@ -4,7 +4,6 @@ import { Variable } from '../../database/entities/Variable'
import { InternalFlowiseError } from '../../errors/internalFlowiseError'
import { getErrorMessage } from '../../errors/utils'
import { getAppVersion } from '../../utils'
import { getWorkspaceSearchOptions } from '../../enterprise/utils/ControllerServiceUtils'
import { QueryRunner } from 'typeorm'
import { validate } from 'uuid'
@@ -44,11 +43,26 @@ const deleteVariable = async (variableId: string): Promise<any> => {
}
}
const getAllVariables = async (workspaceId?: string) => {
const getAllVariables = async (workspaceId?: string, page: number = -1, limit: number = -1) => {
try {
const appServer = getRunningExpressApp()
const dbResponse = await appServer.AppDataSource.getRepository(Variable).findBy(getWorkspaceSearchOptions(workspaceId))
return dbResponse
const queryBuilder = appServer.AppDataSource.getRepository(Variable)
.createQueryBuilder('variable')
.orderBy('variable.updatedDate', 'DESC')
if (page > 0 && limit > 0) {
queryBuilder.skip((page - 1) * limit)
queryBuilder.take(limit)
}
if (workspaceId) queryBuilder.andWhere('variable.workspaceId = :workspaceId', { workspaceId })
const [data, total] = await queryBuilder.getManyAndCount()
if (page > 0 && limit > 0) {
return { data, total }
} else {
return data
}
} catch (error) {
throw new InternalFlowiseError(
StatusCodes.INTERNAL_SERVER_ERROR,
packages/server/src/utils/buildChatflow.ts
+2 -2
View File
@@ -57,7 +57,7 @@ import {
constructGraphs,
getAPIOverrideConfig
} from '../utils'
import { validateChatflowAPIKey } from './validateKey'
import { validateFlowAPIKey } from './validateKey'
import logger from './logger'
import { utilAddChatMessage } from './addChatMesage'
import { checkPredictions, checkStorage, updatePredictionsUsage, updateStorageUsage } from './quotaUsage'
@@ -923,7 +923,7 @@ export const utilBuildChatflow = async (req: Request, isInternal: boolean = fals
try {
// Validate API Key if its external API request
if (!isInternal) {
const isKeyValidated = await validateChatflowAPIKey(req, chatflow)
const isKeyValidated = await validateFlowAPIKey(req, chatflow)
if (!isKeyValidated) {
throw new InternalFlowiseError(StatusCodes.UNAUTHORIZED, `Unauthorized`)
}
packages/server/src/utils/getChatMessage.ts
+252 -55
View File
@@ -4,7 +4,6 @@ import { ChatMessage } from '../database/entities/ChatMessage'
import { ChatMessageFeedback } from '../database/entities/ChatMessageFeedback'
import { ChatFlow } from '../database/entities/ChatFlow'
import { getRunningExpressApp } from '../utils/getRunningExpressApp'
import { aMonthAgo } from '.'
/**
* Method that get chat messages.
@@ -19,6 +18,7 @@ import { aMonthAgo } from '.'
* @param {boolean} feedback
* @param {ChatMessageRatingType[]} feedbackTypes
*/
interface GetChatMessageParams {
chatflowid: string
chatTypes?: ChatType[]
@@ -32,6 +32,8 @@ interface GetChatMessageParams {
feedback?: boolean
feedbackTypes?: ChatMessageRatingType[]
activeWorkspaceId?: string
page?: number
pageSize?: number
}
export const utilGetChatMessage = async ({
@@ -46,72 +48,44 @@ export const utilGetChatMessage = async ({
messageId,
feedback,
feedbackTypes,
activeWorkspaceId
activeWorkspaceId,
page = -1,
pageSize = -1
}: GetChatMessageParams): Promise<ChatMessage[]> => {
if (!page) page = -1
if (!pageSize) pageSize = -1
const appServer = getRunningExpressApp()
// Check if chatflow workspaceId is same as activeWorkspaceId
if (activeWorkspaceId) {
const chatflow = await appServer.AppDataSource.getRepository(ChatFlow).findOneBy({
id: chatflowid
id: chatflowid,
workspaceId: activeWorkspaceId
})
if (chatflow?.workspaceId !== activeWorkspaceId) {
if (!chatflow) {
throw new Error('Unauthorized access')
}
} else {
throw new Error('Unauthorized access')
}
if (feedback) {
const query = await appServer.AppDataSource.getRepository(ChatMessage).createQueryBuilder('chat_message')
// do the join with chat message feedback based on messageId for each chat message in the chatflow
query
.leftJoinAndSelect('chat_message.execution', 'execution')
.leftJoinAndMapOne('chat_message.feedback', ChatMessageFeedback, 'feedback', 'feedback.messageId = chat_message.id')
.where('chat_message.chatflowid = :chatflowid', { chatflowid })
// based on which parameters are available add `andWhere` clauses to the query
if (chatTypes && chatTypes.length > 0) {
query.andWhere('chat_message.chatType IN (:...chatTypes)', { chatTypes })
}
if (chatId) {
query.andWhere('chat_message.chatId = :chatId', { chatId })
}
if (memoryType) {
query.andWhere('chat_message.memoryType = :memoryType', { memoryType })
}
if (sessionId) {
query.andWhere('chat_message.sessionId = :sessionId', { sessionId })
}
// set date range
if (startDate) {
query.andWhere('chat_message.createdDate >= :startDateTime', { startDateTime: startDate ? new Date(startDate) : aMonthAgo() })
}
if (endDate) {
query.andWhere('chat_message.createdDate <= :endDateTime', { endDateTime: endDate ? new Date(endDate) : new Date() })
}
// sort
query.orderBy('chat_message.createdDate', sortOrder === 'DESC' ? 'DESC' : 'ASC')
const messages = (await query.getMany()) as Array<ChatMessage & { feedback: ChatMessageFeedback }>
if (feedbackTypes && feedbackTypes.length > 0) {
// just applying a filter to the messages array will only return the messages that have feedback,
// but we also want the message before the feedback message which is the user message.
const indicesToKeep = new Set()
messages.forEach((message, index) => {
if (message.role === 'apiMessage' && message.feedback && feedbackTypes.includes(message.feedback.rating)) {
if (index > 0) indicesToKeep.add(index - 1)
indicesToKeep.add(index)
}
})
return messages.filter((_, index) => indicesToKeep.has(index))
}
return messages
// Handle feedback queries with improved efficiency
return await handleFeedbackQuery({
chatflowid,
chatTypes,
sortOrder,
chatId,
memoryType,
sessionId,
startDate,
endDate,
messageId,
feedbackTypes,
page,
pageSize
})
}
let createdDateQuery
@@ -146,3 +120,226 @@ export const utilGetChatMessage = async ({
return messages
}
async function handleFeedbackQuery(params: {
chatflowid: string
chatTypes?: ChatType[]
sortOrder: string
chatId?: string
memoryType?: string
sessionId?: string
startDate?: string
endDate?: string
messageId?: string
feedbackTypes?: ChatMessageRatingType[]
page: number
pageSize: number
}): Promise<ChatMessage[]> {
const {
chatflowid,
chatTypes,
sortOrder,
chatId,
memoryType,
sessionId,
startDate,
endDate,
messageId,
feedbackTypes,
page,
pageSize
} = params
const appServer = getRunningExpressApp()
// For specific session/message queries, no pagination needed
if (sessionId || messageId) {
return await getMessagesWithFeedback(params, false)
}
// For paginated queries, handle session-based pagination efficiently
if (page > -1 && pageSize > -1) {
// First get session IDs with pagination
const sessionQuery = appServer.AppDataSource.getRepository(ChatMessage)
.createQueryBuilder('chat_message')
.select('DISTINCT chat_message.sessionId', 'sessionId')
.where('chat_message.chatflowid = :chatflowid', { chatflowid })
// Apply basic filters
if (chatTypes && chatTypes.length > 0) {
sessionQuery.andWhere('chat_message.chatType IN (:...chatTypes)', { chatTypes })
}
if (chatId) {
sessionQuery.andWhere('chat_message.chatId = :chatId', { chatId })
}
if (memoryType) {
sessionQuery.andWhere('chat_message.memoryType = :memoryType', { memoryType })
}
if (startDate && typeof startDate === 'string') {
sessionQuery.andWhere('chat_message.createdDate >= :startDateTime', {
startDateTime: new Date(startDate)
})
}
if (endDate && typeof endDate === 'string') {
sessionQuery.andWhere('chat_message.createdDate <= :endDateTime', {
endDateTime: new Date(endDate)
})
}
// If feedback types are specified, only get sessions with those feedback types
if (feedbackTypes && feedbackTypes.length > 0) {
sessionQuery
.leftJoin(ChatMessageFeedback, 'feedback', 'feedback.messageId = chat_message.id')
.andWhere('feedback.rating IN (:...feedbackTypes)', { feedbackTypes })
}
const startIndex = pageSize * (page - 1)
const sessionIds = await sessionQuery
.orderBy('MAX(chat_message.createdDate)', sortOrder === 'DESC' ? 'DESC' : 'ASC')
.groupBy('chat_message.sessionId')
.offset(startIndex)
.limit(pageSize)
.getRawMany()
if (sessionIds.length === 0) {
return []
}
// Get all messages for these sessions
const sessionIdList = sessionIds.map((s) => s.sessionId)
return await getMessagesWithFeedback(
{
...params,
sessionId: undefined // Clear specific sessionId since we're using list
},
true,
sessionIdList
)
}
// No pagination - get all feedback messages
return await getMessagesWithFeedback(params, false)
}
async function getMessagesWithFeedback(
params: {
chatflowid: string
chatTypes?: ChatType[]
sortOrder: string
chatId?: string
memoryType?: string
sessionId?: string
startDate?: string
endDate?: string
messageId?: string
feedbackTypes?: ChatMessageRatingType[]
},
useSessionList: boolean = false,
sessionIdList?: string[]
): Promise<ChatMessage[]> {
const { chatflowid, chatTypes, sortOrder, chatId, memoryType, sessionId, startDate, endDate, messageId, feedbackTypes } = params
const appServer = getRunningExpressApp()
const query = appServer.AppDataSource.getRepository(ChatMessage).createQueryBuilder('chat_message')
query
.leftJoinAndSelect('chat_message.execution', 'execution')
.leftJoinAndMapOne('chat_message.feedback', ChatMessageFeedback, 'feedback', 'feedback.messageId = chat_message.id')
.where('chat_message.chatflowid = :chatflowid', { chatflowid })
// Apply filters
if (useSessionList && sessionIdList && sessionIdList.length > 0) {
query.andWhere('chat_message.sessionId IN (:...sessionIds)', { sessionIds: sessionIdList })
}
if (chatTypes && chatTypes.length > 0) {
query.andWhere('chat_message.chatType IN (:...chatTypes)', { chatTypes })
}
if (chatId) {
query.andWhere('chat_message.chatId = :chatId', { chatId })
}
if (memoryType) {
query.andWhere('chat_message.memoryType = :memoryType', { memoryType })
}
if (sessionId) {
query.andWhere('chat_message.sessionId = :sessionId', { sessionId })
}
if (messageId) {
query.andWhere('chat_message.id = :messageId', { messageId })
}
if (startDate && typeof startDate === 'string') {
query.andWhere('chat_message.createdDate >= :startDateTime', {
startDateTime: new Date(startDate)
})
}
if (endDate && typeof endDate === 'string') {
query.andWhere('chat_message.createdDate <= :endDateTime', {
endDateTime: new Date(endDate)
})
}
// Pre-filter by feedback types if specified (more efficient than post-processing)
if (feedbackTypes && feedbackTypes.length > 0) {
query.andWhere('(feedback.rating IN (:...feedbackTypes) OR feedback.rating IS NULL)', { feedbackTypes })
}
query.orderBy('chat_message.createdDate', sortOrder === 'DESC' ? 'DESC' : 'ASC')
const messages = (await query.getMany()) as Array<ChatMessage & { feedback: ChatMessageFeedback }>
// Apply feedback type filtering with previous message inclusion
if (feedbackTypes && feedbackTypes.length > 0) {
return filterMessagesWithFeedback(messages, feedbackTypes)
}
return messages
}
function filterMessagesWithFeedback(
messages: Array<ChatMessage & { feedback: ChatMessageFeedback }>,
feedbackTypes: ChatMessageRatingType[]
): ChatMessage[] {
// Group messages by session for proper filtering
const sessionGroups = new Map<string, Array<ChatMessage & { feedback: ChatMessageFeedback }>>()
messages.forEach((message) => {
const sessionId = message.sessionId
if (!sessionId) return // Skip messages without sessionId
if (!sessionGroups.has(sessionId)) {
sessionGroups.set(sessionId, [])
}
sessionGroups.get(sessionId)!.push(message)
})
const result: ChatMessage[] = []
// Process each session group
sessionGroups.forEach((sessionMessages) => {
// Sort by creation date to ensure proper order
sessionMessages.sort((a, b) => new Date(a.createdDate).getTime() - new Date(b.createdDate).getTime())
const toInclude = new Set<number>()
sessionMessages.forEach((message, index) => {
if (message.role === 'apiMessage' && message.feedback && feedbackTypes.includes(message.feedback.rating)) {
// Include the feedback message
toInclude.add(index)
// Include the previous message (user message) if it exists
if (index > 0) {
toInclude.add(index - 1)
}
}
})
// Add filtered messages to result
sessionMessages.forEach((message, index) => {
if (toInclude.has(index)) {
result.push(message)
}
})
})
// Sort final result by creation date
return result.sort((a, b) => new Date(a.createdDate).getTime() - new Date(b.createdDate).getTime())
}
packages/server/src/utils/pagination.ts
+29
View File
@@ -0,0 +1,29 @@
import { InternalFlowiseError } from '../errors/internalFlowiseError'
import { StatusCodes } from 'http-status-codes'
import { Request } from 'express'
type Pagination = {
page: number
limit: number
}
export const getPageAndLimitParams = (req: Request): Pagination => {
// by default assume no pagination
let page = -1
let limit = -1
if (req.query.page) {
// if page is provided, make sure it's a positive number
page = parseInt(req.query.page as string)
if (page < 0) {
throw new InternalFlowiseError(StatusCodes.PRECONDITION_FAILED, `Error: page cannot be negative!`)
}
}
if (req.query.limit) {
// if limit is provided, make sure it's a positive number
limit = parseInt(req.query.limit as string)
if (limit < 0) {
throw new InternalFlowiseError(StatusCodes.PRECONDITION_FAILED, `Error: limit cannot be negative!`)
}
}
return { page, limit }
}
packages/server/src/utils/upsertVector.ts
+2 -2
View File
@@ -21,7 +21,7 @@ import {
getStartingNodes,
getAPIOverrideConfig
} from '../utils'
import { validateChatflowAPIKey } from './validateKey'
import { validateFlowAPIKey } from './validateKey'
import { IncomingInput, INodeDirectedGraph, IReactFlowObject, ChatType, IExecuteFlowParams, MODE } from '../Interface'
import { ChatFlow } from '../database/entities/ChatFlow'
import { getRunningExpressApp } from '../utils/getRunningExpressApp'
@@ -251,7 +251,7 @@ export const upsertVector = async (req: Request, isInternal: boolean = false) =>
const files = (req.files as Express.Multer.File[]) || []
if (!isInternal) {
const isKeyValidated = await validateChatflowAPIKey(req, chatflow)
const isKeyValidated = await validateFlowAPIKey(req, chatflow)
if (!isKeyValidated) {
throw new InternalFlowiseError(StatusCodes.UNAUTHORIZED, `Unauthorized`)
}
packages/server/src/utils/validateKey.ts
+39 -34
View File
@@ -1,14 +1,15 @@
import { Request } from 'express'
import { ChatFlow } from '../database/entities/ChatFlow'
import { ApiKey } from '../database/entities/ApiKey'
import { compareKeys } from './apiKey'
import apikeyService from '../services/apikey'
/**
* Validate Chatflow API Key
* Validate flow API Key, this is needed because Prediction/Upsert API is public
* @param {Request} req
* @param {ChatFlow} chatflow
*/
export const validateChatflowAPIKey = async (req: Request, chatflow: ChatFlow) => {
export const validateFlowAPIKey = async (req: Request, chatflow: ChatFlow): Promise<boolean> => {
const chatFlowApiKeyId = chatflow?.apikeyid
if (!chatFlowApiKeyId) return true
@@ -16,48 +17,52 @@ export const validateChatflowAPIKey = async (req: Request, chatflow: ChatFlow) =
if (chatFlowApiKeyId && !authorizationHeader) return false
const suppliedKey = authorizationHeader.split(`Bearer `).pop()
if (suppliedKey) {
const keys = await apikeyService.getAllApiKeys()
const apiSecret = keys.find((key: any) => key.id === chatFlowApiKeyId)?.apiSecret
if (!apiSecret) return false
if (!compareKeys(apiSecret, suppliedKey)) return false
if (!suppliedKey) return false
try {
const apiKey = await apikeyService.getApiKeyById(chatFlowApiKeyId)
if (!apiKey) return false
const apiKeyWorkSpaceId = apiKey.workspaceId
if (!apiKeyWorkSpaceId) return false
if (apiKeyWorkSpaceId !== chatflow.workspaceId) return false
const apiSecret = apiKey.apiSecret
if (!apiSecret || !compareKeys(apiSecret, suppliedKey)) return false
return true
} catch (error) {
return false
}
return false
}
/**
* Validate API Key
* Validate and Get API Key Information
* @param {Request} req
* @returns {Promise<{isValid: boolean, apiKey?: ApiKey, workspaceId?: string}>}
*/
export const validateAPIKey = async (req: Request) => {
export const validateAPIKey = async (req: Request): Promise<{ isValid: boolean; apiKey?: ApiKey; workspaceId?: string }> => {
const authorizationHeader = (req.headers['Authorization'] as string) ?? (req.headers['authorization'] as string) ?? ''
if (!authorizationHeader) return false
if (!authorizationHeader) return { isValid: false }
const suppliedKey = authorizationHeader.split(`Bearer `).pop()
if (!suppliedKey) return { isValid: false }
if (suppliedKey) {
const keys = await apikeyService.getAllApiKeys()
const apiSecret = keys.find((key: any) => key.apiKey === suppliedKey)?.apiSecret
if (!apiSecret) return false
if (!compareKeys(apiSecret, suppliedKey)) return false
return true
try {
const apiKey = await apikeyService.getApiKey(suppliedKey)
if (!apiKey) return { isValid: false }
const apiKeyWorkSpaceId = apiKey.workspaceId
if (!apiKeyWorkSpaceId) return { isValid: false }
const apiSecret = apiKey.apiSecret
if (!apiSecret || !compareKeys(apiSecret, suppliedKey)) {
return { isValid: false, apiKey, workspaceId: apiKey.workspaceId }
}
return { isValid: true, apiKey, workspaceId: apiKey.workspaceId }
} catch (error) {
return { isValid: false }
}
return false
}
/**
* Get API Key WorkspaceID
* @param {Request} req
*/
export const getAPIKeyWorkspaceID = async (req: Request) => {
const authorizationHeader = (req.headers['Authorization'] as string) ?? (req.headers['authorization'] as string) ?? ''
if (!authorizationHeader) return false
const suppliedKey = authorizationHeader.split(`Bearer `).pop()
if (suppliedKey) {
const key = await apikeyService.getApiKey(suppliedKey)
return key?.workspaceId
}
return undefined
}