New Feature Pagination (#4704)

* common pagination component

* Pagination for Doc Store Dashboard

* Pagination for Executions Dashboard

* Pagination Support for Tables

* lint fixes

* update view message dialog UI

* initial loading was ignoring the pagination counts

* 1) default page size change
2) ensure page limits are passed on load
3) co-pilot review comments (n+1 query)
4)

* 1) default page size change
2) ensure page limits are passed on load
3) co-pilot review comments (n+1 query)
4) refresh lists after insert/delete.

* Enhancement: Improve handling of empty responses in DocumentStore and API key services

- Added check for empty entities in DocumentStoreDTO.fromEntities to return an empty array.
- Updated condition in getAllDocumentStores to handle total count correctly, allowing for zero total.
- Refined logic in getAllApiKeys to check for empty keys and ensure correct API key retrieval.
- Adjusted UI components to safely handle potential undefined apiKeys array.

* Refresh API key list on pagination change

* Enhancement: Update pagination and filter handling across components
- Increased default items per page in AgentExecutions from 10 to 12.
- Improved JSON parsing for chat type and feedback type filters in ViewMessagesDialog.
- Enhanced execution filtering logic in AgentExecutions to ensure proper pagination and state management.
- Refactored filter section in AgentExecutions for better readability and functionality.
- Updated refresh logic in Agentflows to use the correct agentflow version.

* add workspaceId to removeAllChatMessages

* Refactor chat message retrieval logic for improved efficiency and maintainability

- Introduced a new `handleFeedbackQuery` function to streamline feedback-related queries.
- Enhanced pagination handling for session-based queries in `getMessagesWithFeedback`.
- Updated `ViewMessagesDialog` to sort messages in descending order by default.
- Simplified image rendering logic in `DocumentStoreTable` for better readability.

* - Update  `validateChatflowAPIKey` and `validateAPIKey` functions to get the correct keys array
- Enhanced error handling in the `sanitizeExecution` function to ensure safe access to nested properties

* Refactor API key validation logic for improved accuracy and error handling

- Consolidated API key validation in `validateAPIKey` to return detailed validation results.
- Updated `validateFlowAPIKey` to streamline flow API key validation.
- Introduced `getApiKeyById` function in the API key service for better key retrieval.
- Removed unused function `getAllChatSessionsFromChatflow` from the chat message API.

---------

Co-authored-by: Henry <hzj94@hotmail.com>

packages/server/src/controllers/apikey/index.ts

@@ -2,12 +2,14 @@ import { Request, Response, NextFunction } from 'express'
 import { StatusCodes } from 'http-status-codes'
 import { InternalFlowiseError } from '../../errors/internalFlowiseError'
 import apikeyService from '../../services/apikey'
+import { getPageAndLimitParams } from '../../utils/pagination'
 
 // Get api keys
 const getAllApiKeys = async (req: Request, res: Response, next: NextFunction) => {
     try {
         const autoCreateNewKey = true
-        const apiResponse = await apikeyService.getAllApiKeys(req.user?.activeWorkspaceId, autoCreateNewKey)
+        const { page, limit } = getPageAndLimitParams(req)
+        const apiResponse = await apikeyService.getAllApiKeys(req.user?.activeWorkspaceId, autoCreateNewKey, page, limit)
         return res.json(apiResponse)
     } catch (error) {
         next(error)

packages/server/src/controllers/chat-messages/index.ts

@@ -9,6 +9,7 @@ import { ChatMessage } from '../../database/entities/ChatMessage'
 import { InternalFlowiseError } from '../../errors/internalFlowiseError'
 import { StatusCodes } from 'http-status-codes'
 import { utilGetChatMessage } from '../../utils/getChatMessage'
+import { getPageAndLimitParams } from '../../utils/pagination'
 
 const getFeedbackTypeFilters = (_feedbackTypeFilters: ChatMessageRatingType[]): ChatMessageRatingType[] | undefined => {
     try {
@@ -71,6 +72,9 @@ const getAllChatMessages = async (req: Request, res: Response, next: NextFunctio
         const startDate = req.query?.startDate as string | undefined
         const endDate = req.query?.endDate as string | undefined
         const feedback = req.query?.feedback as boolean | undefined
+
+        const { page, limit } = getPageAndLimitParams(req)
+
         let feedbackTypeFilters = req.query?.feedbackType as ChatMessageRatingType[] | undefined
         if (feedbackTypeFilters) {
             feedbackTypeFilters = getFeedbackTypeFilters(feedbackTypeFilters)
@@ -93,7 +97,9 @@ const getAllChatMessages = async (req: Request, res: Response, next: NextFunctio
             messageId,
             feedback,
             feedbackTypeFilters,
-            activeWorkspaceId
+            activeWorkspaceId,
+            page,
+            limit
         )
         return res.json(parseAPIResponse(apiResponse))
     } catch (error) {
@@ -202,7 +208,8 @@ const removeAllChatMessages = async (req: Request, res: Response, next: NextFunc
                 startDate,
                 endDate,
                 feedback: isFeedback,
-                feedbackTypes: feedbackTypeFilters
+                feedbackTypes: feedbackTypeFilters,
+                activeWorkspaceId: workspaceId
             })
             const messageIds = messages.map((message) => message.id)
 

packages/server/src/controllers/chatflows/index.ts

@@ -8,6 +8,7 @@ import chatflowsService from '../../services/chatflows'
 import { getRunningExpressApp } from '../../utils/getRunningExpressApp'
 import { checkUsageLimit } from '../../utils/quotaUsage'
 import { RateLimiterManager } from '../../utils/rateLimit'
+import { getPageAndLimitParams } from '../../utils/pagination'
 
 const checkIfChatflowIsValidForStreaming = async (req: Request, res: Response, next: NextFunction) => {
     try {
@@ -67,7 +68,14 @@ const deleteChatflow = async (req: Request, res: Response, next: NextFunction) =
 
 const getAllChatflows = async (req: Request, res: Response, next: NextFunction) => {
     try {
-        const apiResponse = await chatflowsService.getAllChatflows(req.query?.type as ChatflowType, req.user?.activeWorkspaceId)
+        const { page, limit } = getPageAndLimitParams(req)
+
+        const apiResponse = await chatflowsService.getAllChatflows(
+            req.query?.type as ChatflowType,
+            req.user?.activeWorkspaceId,
+            page,
+            limit
+        )
         return res.json(apiResponse)
     } catch (error) {
         next(error)

packages/server/src/controllers/dataset/index.ts

@@ -2,10 +2,12 @@ import { Request, Response, NextFunction } from 'express'
 import { InternalFlowiseError } from '../../errors/internalFlowiseError'
 import datasetService from '../../services/dataset'
 import { StatusCodes } from 'http-status-codes'
+import { getPageAndLimitParams } from '../../utils/pagination'
 
 const getAllDatasets = async (req: Request, res: Response, next: NextFunction) => {
     try {
-        const apiResponse = await datasetService.getAllDatasets(req.user?.activeWorkspaceId)
+        const { page, limit } = getPageAndLimitParams(req)
+        const apiResponse = await datasetService.getAllDatasets(req.user?.activeWorkspaceId, page, limit)
         return res.json(apiResponse)
     } catch (error) {
         next(error)
@@ -17,7 +19,8 @@ const getDataset = async (req: Request, res: Response, next: NextFunction) => {
         if (typeof req.params === 'undefined' || !req.params.id) {
             throw new InternalFlowiseError(StatusCodes.PRECONDITION_FAILED, `Error: datasetService.getDataset - id not provided!`)
         }
-        const apiResponse = await datasetService.getDataset(req.params.id)
+        const { page, limit } = getPageAndLimitParams(req)
+        const apiResponse = await datasetService.getDataset(req.params.id, page, limit)
         return res.json(apiResponse)
     } catch (error) {
         next(error)

packages/server/src/controllers/documentstore/index.ts

@@ -6,6 +6,7 @@ import { InternalFlowiseError } from '../../errors/internalFlowiseError'
 import { DocumentStoreDTO } from '../../Interface'
 import { getRunningExpressApp } from '../../utils/getRunningExpressApp'
 import { FLOWISE_COUNTER_STATUS, FLOWISE_METRIC_COUNTERS } from '../../Interface.Metrics'
+import { getPageAndLimitParams } from '../../utils/pagination'
 
 const createDocumentStore = async (req: Request, res: Response, next: NextFunction) => {
     try {
@@ -37,8 +38,17 @@ const createDocumentStore = async (req: Request, res: Response, next: NextFuncti
 
 const getAllDocumentStores = async (req: Request, res: Response, next: NextFunction) => {
     try {
-        const apiResponse = await documentStoreService.getAllDocumentStores(req.user?.activeWorkspaceId)
-        return res.json(DocumentStoreDTO.fromEntities(apiResponse))
+        const { page, limit } = getPageAndLimitParams(req)
+
+        const apiResponse: any = await documentStoreService.getAllDocumentStores(req.user?.activeWorkspaceId, page, limit)
+        if (apiResponse?.total >= 0) {
+            return res.json({
+                total: apiResponse.total,
+                data: DocumentStoreDTO.fromEntities(apiResponse.data)
+            })
+        } else {
+            return res.json(DocumentStoreDTO.fromEntities(apiResponse))
+        }
     } catch (error) {
         next(error)
     }

packages/server/src/controllers/evaluations/index.ts

@@ -2,6 +2,7 @@ import { Request, Response, NextFunction } from 'express'
 import { InternalFlowiseError } from '../../errors/internalFlowiseError'
 import { StatusCodes } from 'http-status-codes'
 import evaluationsService from '../../services/evaluations'
+import { getPageAndLimitParams } from '../../utils/pagination'
 
 const createEvaluation = async (req: Request, res: Response, next: NextFunction) => {
     try {
@@ -81,7 +82,8 @@ const deleteEvaluation = async (req: Request, res: Response, next: NextFunction)
 
 const getAllEvaluations = async (req: Request, res: Response, next: NextFunction) => {
     try {
-        const apiResponse = await evaluationsService.getAllEvaluations(req.user?.activeWorkspaceId)
+        const { page, limit } = getPageAndLimitParams(req)
+        const apiResponse = await evaluationsService.getAllEvaluations(req.user?.activeWorkspaceId, page, limit)
         return res.json(apiResponse)
     } catch (error) {
         next(error)

packages/server/src/controllers/evaluators/index.ts

@@ -2,10 +2,12 @@ import { Request, Response, NextFunction } from 'express'
 import { InternalFlowiseError } from '../../errors/internalFlowiseError'
 import { StatusCodes } from 'http-status-codes'
 import evaluatorService from '../../services/evaluator'
+import { getPageAndLimitParams } from '../../utils/pagination'
 
 const getAllEvaluators = async (req: Request, res: Response, next: NextFunction) => {
     try {
-        const apiResponse = await evaluatorService.getAllEvaluators(req.user?.activeWorkspaceId)
+        const { page, limit } = getPageAndLimitParams(req)
+        const apiResponse = await evaluatorService.getAllEvaluators(req.user?.activeWorkspaceId, page, limit)
         return res.json(apiResponse)
     } catch (error) {
         next(error)

packages/server/src/controllers/stats/index.ts

@@ -45,7 +45,16 @@ const getChatflowStats = async (req: Request, res: Response, next: NextFunction)
                 return res.status(500).send(e)
             }
         }
-        const apiResponse = await statsService.getChatflowStats(chatflowid, chatTypes, startDate, endDate, '', true, feedbackTypeFilters)
+        const apiResponse = await statsService.getChatflowStats(
+            chatflowid,
+            chatTypes,
+            startDate,
+            endDate,
+            '',
+            true,
+            feedbackTypeFilters,
+            req.user?.activeWorkspaceId
+        )
         return res.json(apiResponse)
     } catch (error) {
         next(error)

packages/server/src/controllers/tools/index.ts

@@ -2,6 +2,7 @@ import { NextFunction, Request, Response } from 'express'
 import toolsService from '../../services/tools'
 import { InternalFlowiseError } from '../../errors/internalFlowiseError'
 import { StatusCodes } from 'http-status-codes'
+import { getPageAndLimitParams } from '../../utils/pagination'
 
 const createTool = async (req: Request, res: Response, next: NextFunction) => {
     try {
@@ -40,7 +41,8 @@ const deleteTool = async (req: Request, res: Response, next: NextFunction) => {
 
 const getAllTools = async (req: Request, res: Response, next: NextFunction) => {
     try {
-        const apiResponse = await toolsService.getAllTools(req.user?.activeWorkspaceId)
+        const { page, limit } = getPageAndLimitParams(req)
+        const apiResponse = await toolsService.getAllTools(req.user?.activeWorkspaceId, page, limit)
         return res.json(apiResponse)
     } catch (error) {
         next(error)

packages/server/src/controllers/variables/index.ts

@@ -3,6 +3,7 @@ import variablesService from '../../services/variables'
 import { Variable } from '../../database/entities/Variable'
 import { InternalFlowiseError } from '../../errors/internalFlowiseError'
 import { StatusCodes } from 'http-status-codes'
+import { getPageAndLimitParams } from '../../utils/pagination'
 
 const createVariable = async (req: Request, res: Response, next: NextFunction) => {
     try {
@@ -45,7 +46,8 @@ const deleteVariable = async (req: Request, res: Response, next: NextFunction) =
 
 const getAllVariables = async (req: Request, res: Response, next: NextFunction) => {
     try {
-        const apiResponse = await variablesService.getAllVariables(req.user?.activeWorkspaceId)
+        const { page, limit } = getPageAndLimitParams(req)
+        const apiResponse = await variablesService.getAllVariables(req.user?.activeWorkspaceId, page, limit)
         return res.json(apiResponse)
     } catch (error) {
         next(error)