New Feature Pagination (#4704)

* common pagination component

* Pagination for Doc Store Dashboard

* Pagination for Executions Dashboard

* Pagination Support for Tables

* lint fixes

* update view message dialog UI

* initial loading was ignoring the pagination counts

* 1) default page size change
2) ensure page limits are passed on load
3) co-pilot review comments (n+1 query)
4)

* 1) default page size change
2) ensure page limits are passed on load
3) co-pilot review comments (n+1 query)
4) refresh lists after insert/delete.

* Enhancement: Improve handling of empty responses in DocumentStore and API key services

- Added check for empty entities in DocumentStoreDTO.fromEntities to return an empty array.
- Updated condition in getAllDocumentStores to handle total count correctly, allowing for zero total.
- Refined logic in getAllApiKeys to check for empty keys and ensure correct API key retrieval.
- Adjusted UI components to safely handle potential undefined apiKeys array.

* Refresh API key list on pagination change

* Enhancement: Update pagination and filter handling across components
- Increased default items per page in AgentExecutions from 10 to 12.
- Improved JSON parsing for chat type and feedback type filters in ViewMessagesDialog.
- Enhanced execution filtering logic in AgentExecutions to ensure proper pagination and state management.
- Refactored filter section in AgentExecutions for better readability and functionality.
- Updated refresh logic in Agentflows to use the correct agentflow version.

* add workspaceId to removeAllChatMessages

* Refactor chat message retrieval logic for improved efficiency and maintainability

- Introduced a new `handleFeedbackQuery` function to streamline feedback-related queries.
- Enhanced pagination handling for session-based queries in `getMessagesWithFeedback`.
- Updated `ViewMessagesDialog` to sort messages in descending order by default.
- Simplified image rendering logic in `DocumentStoreTable` for better readability.

* - Update  `validateChatflowAPIKey` and `validateAPIKey` functions to get the correct keys array
- Enhanced error handling in the `sanitizeExecution` function to ensure safe access to nested properties

* Refactor API key validation logic for improved accuracy and error handling

- Consolidated API key validation in `validateAPIKey` to return detailed validation results.
- Updated `validateFlowAPIKey` to streamline flow API key validation.
- Introduced `getApiKeyById` function in the API key service for better key retrieval.
- Removed unused function `getAllChatSessionsFromChatflow` from the chat message API.

---------

Co-authored-by: Henry <hzj94@hotmail.com>
This commit is contained in:
Vinod Kiran
2025-07-10 20:29:24 +05:30
committed by GitHub
parent 6baec93860
commit bf05f25f7e
55 changed files with 2595 additions and 1560 deletions
+2 -2
View File
@@ -57,7 +57,7 @@ import {
constructGraphs,
getAPIOverrideConfig
} from '../utils'
import { validateChatflowAPIKey } from './validateKey'
import { validateFlowAPIKey } from './validateKey'
import logger from './logger'
import { utilAddChatMessage } from './addChatMesage'
import { checkPredictions, checkStorage, updatePredictionsUsage, updateStorageUsage } from './quotaUsage'
@@ -923,7 +923,7 @@ export const utilBuildChatflow = async (req: Request, isInternal: boolean = fals
try {
// Validate API Key if its external API request
if (!isInternal) {
const isKeyValidated = await validateChatflowAPIKey(req, chatflow)
const isKeyValidated = await validateFlowAPIKey(req, chatflow)
if (!isKeyValidated) {
throw new InternalFlowiseError(StatusCodes.UNAUTHORIZED, `Unauthorized`)
}
+252 -55
View File
@@ -4,7 +4,6 @@ import { ChatMessage } from '../database/entities/ChatMessage'
import { ChatMessageFeedback } from '../database/entities/ChatMessageFeedback'
import { ChatFlow } from '../database/entities/ChatFlow'
import { getRunningExpressApp } from '../utils/getRunningExpressApp'
import { aMonthAgo } from '.'
/**
* Method that get chat messages.
@@ -19,6 +18,7 @@ import { aMonthAgo } from '.'
* @param {boolean} feedback
* @param {ChatMessageRatingType[]} feedbackTypes
*/
interface GetChatMessageParams {
chatflowid: string
chatTypes?: ChatType[]
@@ -32,6 +32,8 @@ interface GetChatMessageParams {
feedback?: boolean
feedbackTypes?: ChatMessageRatingType[]
activeWorkspaceId?: string
page?: number
pageSize?: number
}
export const utilGetChatMessage = async ({
@@ -46,72 +48,44 @@ export const utilGetChatMessage = async ({
messageId,
feedback,
feedbackTypes,
activeWorkspaceId
activeWorkspaceId,
page = -1,
pageSize = -1
}: GetChatMessageParams): Promise<ChatMessage[]> => {
if (!page) page = -1
if (!pageSize) pageSize = -1
const appServer = getRunningExpressApp()
// Check if chatflow workspaceId is same as activeWorkspaceId
if (activeWorkspaceId) {
const chatflow = await appServer.AppDataSource.getRepository(ChatFlow).findOneBy({
id: chatflowid
id: chatflowid,
workspaceId: activeWorkspaceId
})
if (chatflow?.workspaceId !== activeWorkspaceId) {
if (!chatflow) {
throw new Error('Unauthorized access')
}
} else {
throw new Error('Unauthorized access')
}
if (feedback) {
const query = await appServer.AppDataSource.getRepository(ChatMessage).createQueryBuilder('chat_message')
// do the join with chat message feedback based on messageId for each chat message in the chatflow
query
.leftJoinAndSelect('chat_message.execution', 'execution')
.leftJoinAndMapOne('chat_message.feedback', ChatMessageFeedback, 'feedback', 'feedback.messageId = chat_message.id')
.where('chat_message.chatflowid = :chatflowid', { chatflowid })
// based on which parameters are available add `andWhere` clauses to the query
if (chatTypes && chatTypes.length > 0) {
query.andWhere('chat_message.chatType IN (:...chatTypes)', { chatTypes })
}
if (chatId) {
query.andWhere('chat_message.chatId = :chatId', { chatId })
}
if (memoryType) {
query.andWhere('chat_message.memoryType = :memoryType', { memoryType })
}
if (sessionId) {
query.andWhere('chat_message.sessionId = :sessionId', { sessionId })
}
// set date range
if (startDate) {
query.andWhere('chat_message.createdDate >= :startDateTime', { startDateTime: startDate ? new Date(startDate) : aMonthAgo() })
}
if (endDate) {
query.andWhere('chat_message.createdDate <= :endDateTime', { endDateTime: endDate ? new Date(endDate) : new Date() })
}
// sort
query.orderBy('chat_message.createdDate', sortOrder === 'DESC' ? 'DESC' : 'ASC')
const messages = (await query.getMany()) as Array<ChatMessage & { feedback: ChatMessageFeedback }>
if (feedbackTypes && feedbackTypes.length > 0) {
// just applying a filter to the messages array will only return the messages that have feedback,
// but we also want the message before the feedback message which is the user message.
const indicesToKeep = new Set()
messages.forEach((message, index) => {
if (message.role === 'apiMessage' && message.feedback && feedbackTypes.includes(message.feedback.rating)) {
if (index > 0) indicesToKeep.add(index - 1)
indicesToKeep.add(index)
}
})
return messages.filter((_, index) => indicesToKeep.has(index))
}
return messages
// Handle feedback queries with improved efficiency
return await handleFeedbackQuery({
chatflowid,
chatTypes,
sortOrder,
chatId,
memoryType,
sessionId,
startDate,
endDate,
messageId,
feedbackTypes,
page,
pageSize
})
}
let createdDateQuery
@@ -146,3 +120,226 @@ export const utilGetChatMessage = async ({
return messages
}
async function handleFeedbackQuery(params: {
chatflowid: string
chatTypes?: ChatType[]
sortOrder: string
chatId?: string
memoryType?: string
sessionId?: string
startDate?: string
endDate?: string
messageId?: string
feedbackTypes?: ChatMessageRatingType[]
page: number
pageSize: number
}): Promise<ChatMessage[]> {
const {
chatflowid,
chatTypes,
sortOrder,
chatId,
memoryType,
sessionId,
startDate,
endDate,
messageId,
feedbackTypes,
page,
pageSize
} = params
const appServer = getRunningExpressApp()
// For specific session/message queries, no pagination needed
if (sessionId || messageId) {
return await getMessagesWithFeedback(params, false)
}
// For paginated queries, handle session-based pagination efficiently
if (page > -1 && pageSize > -1) {
// First get session IDs with pagination
const sessionQuery = appServer.AppDataSource.getRepository(ChatMessage)
.createQueryBuilder('chat_message')
.select('DISTINCT chat_message.sessionId', 'sessionId')
.where('chat_message.chatflowid = :chatflowid', { chatflowid })
// Apply basic filters
if (chatTypes && chatTypes.length > 0) {
sessionQuery.andWhere('chat_message.chatType IN (:...chatTypes)', { chatTypes })
}
if (chatId) {
sessionQuery.andWhere('chat_message.chatId = :chatId', { chatId })
}
if (memoryType) {
sessionQuery.andWhere('chat_message.memoryType = :memoryType', { memoryType })
}
if (startDate && typeof startDate === 'string') {
sessionQuery.andWhere('chat_message.createdDate >= :startDateTime', {
startDateTime: new Date(startDate)
})
}
if (endDate && typeof endDate === 'string') {
sessionQuery.andWhere('chat_message.createdDate <= :endDateTime', {
endDateTime: new Date(endDate)
})
}
// If feedback types are specified, only get sessions with those feedback types
if (feedbackTypes && feedbackTypes.length > 0) {
sessionQuery
.leftJoin(ChatMessageFeedback, 'feedback', 'feedback.messageId = chat_message.id')
.andWhere('feedback.rating IN (:...feedbackTypes)', { feedbackTypes })
}
const startIndex = pageSize * (page - 1)
const sessionIds = await sessionQuery
.orderBy('MAX(chat_message.createdDate)', sortOrder === 'DESC' ? 'DESC' : 'ASC')
.groupBy('chat_message.sessionId')
.offset(startIndex)
.limit(pageSize)
.getRawMany()
if (sessionIds.length === 0) {
return []
}
// Get all messages for these sessions
const sessionIdList = sessionIds.map((s) => s.sessionId)
return await getMessagesWithFeedback(
{
...params,
sessionId: undefined // Clear specific sessionId since we're using list
},
true,
sessionIdList
)
}
// No pagination - get all feedback messages
return await getMessagesWithFeedback(params, false)
}
async function getMessagesWithFeedback(
params: {
chatflowid: string
chatTypes?: ChatType[]
sortOrder: string
chatId?: string
memoryType?: string
sessionId?: string
startDate?: string
endDate?: string
messageId?: string
feedbackTypes?: ChatMessageRatingType[]
},
useSessionList: boolean = false,
sessionIdList?: string[]
): Promise<ChatMessage[]> {
const { chatflowid, chatTypes, sortOrder, chatId, memoryType, sessionId, startDate, endDate, messageId, feedbackTypes } = params
const appServer = getRunningExpressApp()
const query = appServer.AppDataSource.getRepository(ChatMessage).createQueryBuilder('chat_message')
query
.leftJoinAndSelect('chat_message.execution', 'execution')
.leftJoinAndMapOne('chat_message.feedback', ChatMessageFeedback, 'feedback', 'feedback.messageId = chat_message.id')
.where('chat_message.chatflowid = :chatflowid', { chatflowid })
// Apply filters
if (useSessionList && sessionIdList && sessionIdList.length > 0) {
query.andWhere('chat_message.sessionId IN (:...sessionIds)', { sessionIds: sessionIdList })
}
if (chatTypes && chatTypes.length > 0) {
query.andWhere('chat_message.chatType IN (:...chatTypes)', { chatTypes })
}
if (chatId) {
query.andWhere('chat_message.chatId = :chatId', { chatId })
}
if (memoryType) {
query.andWhere('chat_message.memoryType = :memoryType', { memoryType })
}
if (sessionId) {
query.andWhere('chat_message.sessionId = :sessionId', { sessionId })
}
if (messageId) {
query.andWhere('chat_message.id = :messageId', { messageId })
}
if (startDate && typeof startDate === 'string') {
query.andWhere('chat_message.createdDate >= :startDateTime', {
startDateTime: new Date(startDate)
})
}
if (endDate && typeof endDate === 'string') {
query.andWhere('chat_message.createdDate <= :endDateTime', {
endDateTime: new Date(endDate)
})
}
// Pre-filter by feedback types if specified (more efficient than post-processing)
if (feedbackTypes && feedbackTypes.length > 0) {
query.andWhere('(feedback.rating IN (:...feedbackTypes) OR feedback.rating IS NULL)', { feedbackTypes })
}
query.orderBy('chat_message.createdDate', sortOrder === 'DESC' ? 'DESC' : 'ASC')
const messages = (await query.getMany()) as Array<ChatMessage & { feedback: ChatMessageFeedback }>
// Apply feedback type filtering with previous message inclusion
if (feedbackTypes && feedbackTypes.length > 0) {
return filterMessagesWithFeedback(messages, feedbackTypes)
}
return messages
}
function filterMessagesWithFeedback(
messages: Array<ChatMessage & { feedback: ChatMessageFeedback }>,
feedbackTypes: ChatMessageRatingType[]
): ChatMessage[] {
// Group messages by session for proper filtering
const sessionGroups = new Map<string, Array<ChatMessage & { feedback: ChatMessageFeedback }>>()
messages.forEach((message) => {
const sessionId = message.sessionId
if (!sessionId) return // Skip messages without sessionId
if (!sessionGroups.has(sessionId)) {
sessionGroups.set(sessionId, [])
}
sessionGroups.get(sessionId)!.push(message)
})
const result: ChatMessage[] = []
// Process each session group
sessionGroups.forEach((sessionMessages) => {
// Sort by creation date to ensure proper order
sessionMessages.sort((a, b) => new Date(a.createdDate).getTime() - new Date(b.createdDate).getTime())
const toInclude = new Set<number>()
sessionMessages.forEach((message, index) => {
if (message.role === 'apiMessage' && message.feedback && feedbackTypes.includes(message.feedback.rating)) {
// Include the feedback message
toInclude.add(index)
// Include the previous message (user message) if it exists
if (index > 0) {
toInclude.add(index - 1)
}
}
})
// Add filtered messages to result
sessionMessages.forEach((message, index) => {
if (toInclude.has(index)) {
result.push(message)
}
})
})
// Sort final result by creation date
return result.sort((a, b) => new Date(a.createdDate).getTime() - new Date(b.createdDate).getTime())
}
+29
View File
@@ -0,0 +1,29 @@
import { InternalFlowiseError } from '../errors/internalFlowiseError'
import { StatusCodes } from 'http-status-codes'
import { Request } from 'express'
type Pagination = {
page: number
limit: number
}
export const getPageAndLimitParams = (req: Request): Pagination => {
// by default assume no pagination
let page = -1
let limit = -1
if (req.query.page) {
// if page is provided, make sure it's a positive number
page = parseInt(req.query.page as string)
if (page < 0) {
throw new InternalFlowiseError(StatusCodes.PRECONDITION_FAILED, `Error: page cannot be negative!`)
}
}
if (req.query.limit) {
// if limit is provided, make sure it's a positive number
limit = parseInt(req.query.limit as string)
if (limit < 0) {
throw new InternalFlowiseError(StatusCodes.PRECONDITION_FAILED, `Error: limit cannot be negative!`)
}
}
return { page, limit }
}
+2 -2
View File
@@ -21,7 +21,7 @@ import {
getStartingNodes,
getAPIOverrideConfig
} from '../utils'
import { validateChatflowAPIKey } from './validateKey'
import { validateFlowAPIKey } from './validateKey'
import { IncomingInput, INodeDirectedGraph, IReactFlowObject, ChatType, IExecuteFlowParams, MODE } from '../Interface'
import { ChatFlow } from '../database/entities/ChatFlow'
import { getRunningExpressApp } from '../utils/getRunningExpressApp'
@@ -251,7 +251,7 @@ export const upsertVector = async (req: Request, isInternal: boolean = false) =>
const files = (req.files as Express.Multer.File[]) || []
if (!isInternal) {
const isKeyValidated = await validateChatflowAPIKey(req, chatflow)
const isKeyValidated = await validateFlowAPIKey(req, chatflow)
if (!isKeyValidated) {
throw new InternalFlowiseError(StatusCodes.UNAUTHORIZED, `Unauthorized`)
}
+39 -34
View File
@@ -1,14 +1,15 @@
import { Request } from 'express'
import { ChatFlow } from '../database/entities/ChatFlow'
import { ApiKey } from '../database/entities/ApiKey'
import { compareKeys } from './apiKey'
import apikeyService from '../services/apikey'
/**
* Validate Chatflow API Key
* Validate flow API Key, this is needed because Prediction/Upsert API is public
* @param {Request} req
* @param {ChatFlow} chatflow
*/
export const validateChatflowAPIKey = async (req: Request, chatflow: ChatFlow) => {
export const validateFlowAPIKey = async (req: Request, chatflow: ChatFlow): Promise<boolean> => {
const chatFlowApiKeyId = chatflow?.apikeyid
if (!chatFlowApiKeyId) return true
@@ -16,48 +17,52 @@ export const validateChatflowAPIKey = async (req: Request, chatflow: ChatFlow) =
if (chatFlowApiKeyId && !authorizationHeader) return false
const suppliedKey = authorizationHeader.split(`Bearer `).pop()
if (suppliedKey) {
const keys = await apikeyService.getAllApiKeys()
const apiSecret = keys.find((key: any) => key.id === chatFlowApiKeyId)?.apiSecret
if (!apiSecret) return false
if (!compareKeys(apiSecret, suppliedKey)) return false
if (!suppliedKey) return false
try {
const apiKey = await apikeyService.getApiKeyById(chatFlowApiKeyId)
if (!apiKey) return false
const apiKeyWorkSpaceId = apiKey.workspaceId
if (!apiKeyWorkSpaceId) return false
if (apiKeyWorkSpaceId !== chatflow.workspaceId) return false
const apiSecret = apiKey.apiSecret
if (!apiSecret || !compareKeys(apiSecret, suppliedKey)) return false
return true
} catch (error) {
return false
}
return false
}
/**
* Validate API Key
* Validate and Get API Key Information
* @param {Request} req
* @returns {Promise<{isValid: boolean, apiKey?: ApiKey, workspaceId?: string}>}
*/
export const validateAPIKey = async (req: Request) => {
export const validateAPIKey = async (req: Request): Promise<{ isValid: boolean; apiKey?: ApiKey; workspaceId?: string }> => {
const authorizationHeader = (req.headers['Authorization'] as string) ?? (req.headers['authorization'] as string) ?? ''
if (!authorizationHeader) return false
if (!authorizationHeader) return { isValid: false }
const suppliedKey = authorizationHeader.split(`Bearer `).pop()
if (!suppliedKey) return { isValid: false }
if (suppliedKey) {
const keys = await apikeyService.getAllApiKeys()
const apiSecret = keys.find((key: any) => key.apiKey === suppliedKey)?.apiSecret
if (!apiSecret) return false
if (!compareKeys(apiSecret, suppliedKey)) return false
return true
try {
const apiKey = await apikeyService.getApiKey(suppliedKey)
if (!apiKey) return { isValid: false }
const apiKeyWorkSpaceId = apiKey.workspaceId
if (!apiKeyWorkSpaceId) return { isValid: false }
const apiSecret = apiKey.apiSecret
if (!apiSecret || !compareKeys(apiSecret, suppliedKey)) {
return { isValid: false, apiKey, workspaceId: apiKey.workspaceId }
}
return { isValid: true, apiKey, workspaceId: apiKey.workspaceId }
} catch (error) {
return { isValid: false }
}
return false
}
/**
* Get API Key WorkspaceID
* @param {Request} req
*/
export const getAPIKeyWorkspaceID = async (req: Request) => {
const authorizationHeader = (req.headers['Authorization'] as string) ?? (req.headers['authorization'] as string) ?? ''
if (!authorizationHeader) return false
const suppliedKey = authorizationHeader.split(`Bearer `).pop()
if (suppliedKey) {
const key = await apikeyService.getApiKey(suppliedKey)
return key?.workspaceId
}
return undefined
}