Chore/refractor (#4454)

* markdown files and env examples cleanup

* components update

* update jsonlines description

* server refractor

* update telemetry

* add execute custom node

* add ui refractor

* add username and password authenticate

* correctly retrieve past images in agentflowv2

* disable e2e temporarily

* add existing username and password authenticate

* update migration to default workspace

* update todo

* blob storage migrating

* throw error on agent tool call error

* add missing execution import

* add referral

* chore: add error message when importData is undefined

* migrate api keys to db

* fix: data too long for column executionData

* migrate api keys from json to db at init

* add info on account setup

* update docstore missing fields

---------

Co-authored-by: chungyau97 <chungyau97@gmail.com>

packages/server/src/queue/PredictionQueue.ts

@@ -7,6 +7,7 @@ import { RedisEventPublisher } from './RedisEventPublisher'
 import { AbortControllerPool } from '../AbortControllerPool'
 import { BaseQueue } from './BaseQueue'
 import { RedisOptions } from 'bullmq'
+import { UsageCacheManager } from '../UsageCacheManager'
 import logger from '../utils/logger'
 import { generateAgentflowv2 as generateAgentflowv2_json } from 'flowise-components'
 import { databaseEntities } from '../utils'
@@ -18,6 +19,7 @@ interface PredictionQueueOptions {
     cachePool: CachePool
     componentNodes: IComponentNodes
     abortControllerPool: AbortControllerPool
+    usageCacheManager: UsageCacheManager
 }
 
 interface IGenerateAgentflowv2Params extends IExecuteFlowParams {
@@ -35,6 +37,7 @@ export class PredictionQueue extends BaseQueue {
     private cachePool: CachePool
     private appDataSource: DataSource
     private abortControllerPool: AbortControllerPool
+    private usageCacheManager: UsageCacheManager
     private redisPublisher: RedisEventPublisher
     private queueName: string
 
@@ -46,6 +49,7 @@ export class PredictionQueue extends BaseQueue {
         this.cachePool = options.cachePool
         this.appDataSource = options.appDataSource
         this.abortControllerPool = options.abortControllerPool
+        this.usageCacheManager = options.usageCacheManager
         this.redisPublisher = new RedisEventPublisher()
         this.redisPublisher.connect()
     }
@@ -62,6 +66,7 @@ export class PredictionQueue extends BaseQueue {
         if (this.appDataSource) data.appDataSource = this.appDataSource
         if (this.telemetry) data.telemetry = this.telemetry
         if (this.cachePool) data.cachePool = this.cachePool
+        if (this.usageCacheManager) data.usageCacheManager = this.usageCacheManager
         if (this.componentNodes) data.componentNodes = this.componentNodes
         if (this.redisPublisher) data.sseStreamer = this.redisPublisher
 
@@ -78,7 +83,7 @@ export class PredictionQueue extends BaseQueue {
 
         if (Object.prototype.hasOwnProperty.call(data, 'isExecuteCustomFunction')) {
             const executeCustomFunctionData = data as any
-            logger.info(`Executing Custom Function...`)
+            logger.info(`[${executeCustomFunctionData.orgId}]: Executing Custom Function...`)
             return await executeCustomNodeFunction({
                 appDataSource: this.appDataSource,
                 componentNodes: this.componentNodes,

packages/server/src/queue/QueueManager.ts

@@ -10,6 +10,7 @@ import { QueueEventsProducer, RedisOptions } from 'bullmq'
 import { createBullBoard } from 'bull-board'
 import { BullMQAdapter } from 'bull-board/bullMQAdapter'
 import { Express } from 'express'
+import { UsageCacheManager } from '../UsageCacheManager'
 
 const QUEUE_NAME = process.env.QUEUE_NAME || 'flowise-queue'
 
@@ -96,13 +97,15 @@ export class QueueManager {
         telemetry,
         cachePool,
         appDataSource,
-        abortControllerPool
+        abortControllerPool,
+        usageCacheManager
     }: {
         componentNodes: IComponentNodes
         telemetry: Telemetry
         cachePool: CachePool
         appDataSource: DataSource
         abortControllerPool: AbortControllerPool
+        usageCacheManager: UsageCacheManager
     }) {
         const predictionQueueName = `${QUEUE_NAME}-prediction`
         const predictionQueue = new PredictionQueue(predictionQueueName, this.connection, {
@@ -110,7 +113,8 @@ export class QueueManager {
             telemetry,
             cachePool,
             appDataSource,
-            abortControllerPool
+            abortControllerPool,
+            usageCacheManager
         })
         this.registerQueue('prediction', predictionQueue)
         this.predictionQueueEventsProducer = new QueueEventsProducer(predictionQueue.getQueueName(), {
@@ -122,7 +126,8 @@ export class QueueManager {
             componentNodes,
             telemetry,
             cachePool,
-            appDataSource
+            appDataSource,
+            usageCacheManager
         })
         this.registerQueue('upsert', upsertionQueue)
 

packages/server/src/queue/UpsertQueue.ts

@@ -14,11 +14,13 @@ import { executeUpsert } from '../utils/upsertVector'
 import { executeDocStoreUpsert, insertIntoVectorStore, previewChunks, processLoader } from '../services/documentstore'
 import { RedisOptions } from 'bullmq'
 import logger from '../utils/logger'
+import { UsageCacheManager } from '../UsageCacheManager'
 
 interface UpsertQueueOptions {
     appDataSource: DataSource
     telemetry: Telemetry
     cachePool: CachePool
+    usageCacheManager: UsageCacheManager
     componentNodes: IComponentNodes
 }
 
@@ -27,6 +29,7 @@ export class UpsertQueue extends BaseQueue {
     private telemetry: Telemetry
     private cachePool: CachePool
     private appDataSource: DataSource
+    private usageCacheManager: UsageCacheManager
     private queueName: string
 
     constructor(name: string, connection: RedisOptions, options: UpsertQueueOptions) {
@@ -36,6 +39,7 @@ export class UpsertQueue extends BaseQueue {
         this.telemetry = options.telemetry
         this.cachePool = options.cachePool
         this.appDataSource = options.appDataSource
+        this.usageCacheManager = options.usageCacheManager
     }
 
     public getQueueName() {
@@ -52,6 +56,7 @@ export class UpsertQueue extends BaseQueue {
         if (this.appDataSource) data.appDataSource = this.appDataSource
         if (this.telemetry) data.telemetry = this.telemetry
         if (this.cachePool) data.cachePool = this.cachePool
+        if (this.usageCacheManager) data.usageCacheManager = this.usageCacheManager
         if (this.componentNodes) data.componentNodes = this.componentNodes
 
         // document-store/loader/preview