Merge branch 'main' of https://github.com/use-the-fork/Flowise into Bug/rework-redis-connection

# Conflicts:
#	packages/components/package.json
#	packages/ui/src/ui-component/dialog/ViewMessagesDialog.jsx
#	packages/ui/src/views/apikey/index.jsx
#	packages/ui/src/views/assistants/AssistantDialog.jsx
#	packages/ui/src/views/chatflows/index.jsx
#	packages/ui/src/views/credentials/index.jsx

packages/server/marketplaces/chatflows/Claude LLM.json

@@ -1,5 +1,5 @@
 {
-    "description": "Use Anthropic Claude with 100k context window to ingest whole document for QnA",
+    "description": "Use Anthropic Claude with 200k context window to ingest whole document for QnA",
     "nodes": [
         {
             "width": 300,
@@ -148,7 +148,7 @@
                 "id": "chatAnthropic_0",
                 "label": "ChatAnthropic",
                 "name": "chatAnthropic",
-                "version": 2,
+                "version": 3,
                 "type": "ChatAnthropic",
                 "baseClasses": ["ChatAnthropic", "BaseChatModel", "BaseLanguageModel"],
                 "category": "Chat Models",
@@ -171,6 +171,11 @@
                                 "name": "claude-2",
                                 "description": "Claude 2 latest major version, automatically get updates to the model as they are released"
                             },
+                            {
+                                "label": "claude-2.1",
+                                "name": "claude-2.1",
+                                "description": "Claude 2 latest full version"
+                            },
                             {
                                 "label": "claude-instant-1",
                                 "name": "claude-instant-1",
@@ -268,7 +273,7 @@
                     }
                 ],
                 "inputs": {
-                    "modelName": "claude-2",
+                    "modelName": "claude-2.1",
                     "temperature": 0.9,
                     "maxTokensToSample": "",
                     "topP": "",

packages/server/src/Interface.ts

@@ -30,6 +30,7 @@ export interface IChatMessage {
     chatflowid: string
     sourceDocuments?: string
     usedTools?: string
+    fileAnnotations?: string
     chatType: string
     chatId: string
     memoryType?: string

packages/server/src/database/entities/ChatFlow.ts

@@ -36,4 +36,7 @@ export class ChatFlow implements IChatFlow {
 
     @UpdateDateColumn()
     updatedDate: Date
+
+    @Column({ nullable: true, type: 'text' })
+    category?: string
 }

packages/server/src/database/entities/ChatMessage.ts

@@ -23,6 +23,9 @@ export class ChatMessage implements IChatMessage {
     @Column({ nullable: true, type: 'text' })
     usedTools?: string
 
+    @Column({ nullable: true, type: 'text' })
+    fileAnnotations?: string
+
     @Column()
     chatType: string
 

packages/server/src/database/migrations/mysql/1699900910291-AddCategoryToChatFlow.ts

@@ -0,0 +1,12 @@
+import { MigrationInterface, QueryRunner } from 'typeorm'
+
+export class AddCategoryToChatFlow1699900910291 implements MigrationInterface {
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        const columnExists = await queryRunner.hasColumn('chat_flow', 'category')
+        if (!columnExists) queryRunner.query(`ALTER TABLE \`chat_flow\` ADD COLUMN \`category\` TEXT;`)
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE \`chat_flow\` DROP COLUMN \`category\`;`)
+    }
+}

packages/server/src/database/migrations/mysql/1700271021237-AddFileAnnotationsToChatMessage.ts

@@ -0,0 +1,12 @@
+import { MigrationInterface, QueryRunner } from 'typeorm'
+
+export class AddFileAnnotationsToChatMessage1700271021237 implements MigrationInterface {
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        const columnExists = await queryRunner.hasColumn('chat_message', 'fileAnnotations')
+        if (!columnExists) queryRunner.query(`ALTER TABLE \`chat_message\` ADD COLUMN \`fileAnnotations\` TEXT;`)
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE \`chat_message\` DROP COLUMN \`fileAnnotations\`;`)
+    }
+}

packages/server/src/database/migrations/mysql/index.ts

@@ -8,6 +8,8 @@ import { AddAnalytic1694432361423 } from './1694432361423-AddAnalytic'
 import { AddChatHistory1694658767766 } from './1694658767766-AddChatHistory'
 import { AddAssistantEntity1699325775451 } from './1699325775451-AddAssistantEntity'
 import { AddUsedToolsToChatMessage1699481607341 } from './1699481607341-AddUsedToolsToChatMessage'
+import { AddCategoryToChatFlow1699900910291 } from './1699900910291-AddCategoryToChatFlow'
+import { AddFileAnnotationsToChatMessage1700271021237 } from './1700271021237-AddFileAnnotationsToChatMessage'
 
 export const mysqlMigrations = [
     Init1693840429259,
@@ -19,5 +21,7 @@ export const mysqlMigrations = [
     AddAnalytic1694432361423,
     AddChatHistory1694658767766,
     AddAssistantEntity1699325775451,
-    AddUsedToolsToChatMessage1699481607341
+    AddUsedToolsToChatMessage1699481607341,
+    AddCategoryToChatFlow1699900910291,
+    AddFileAnnotationsToChatMessage1700271021237
 ]

packages/server/src/database/migrations/postgres/1699481607341-AddUsedToolsToChatMessage.ts

@@ -6,6 +6,6 @@ export class AddUsedToolsToChatMessage1699481607341 implements MigrationInterfac
     }
 
     public async down(queryRunner: QueryRunner): Promise<void> {
-        await queryRunner.query(`ALTER TABLE "chat_flow" DROP COLUMN "usedTools";`)
+        await queryRunner.query(`ALTER TABLE "chat_message" DROP COLUMN "usedTools";`)
     }
 }

packages/server/src/database/migrations/postgres/1699900910291-AddCategoryToChatFlow.ts

@@ -0,0 +1,11 @@
+import { MigrationInterface, QueryRunner } from 'typeorm'
+
+export class AddCategoryToChatFlow1699900910291 implements MigrationInterface {
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE "chat_flow" ADD COLUMN IF NOT EXISTS "category" TEXT;`)
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE "chat_flow" DROP COLUMN "category";`)
+    }
+}

packages/server/src/database/migrations/postgres/1700271021237-AddFileAnnotationsToChatMessage.ts

@@ -0,0 +1,11 @@
+import { MigrationInterface, QueryRunner } from 'typeorm'
+
+export class AddFileAnnotationsToChatMessage1700271021237 implements MigrationInterface {
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE "chat_message" ADD COLUMN IF NOT EXISTS "fileAnnotations" TEXT;`)
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE "chat_message" DROP COLUMN "fileAnnotations";`)
+    }
+}

packages/server/src/database/migrations/postgres/index.ts

@@ -8,6 +8,8 @@ import { AddAnalytic1694432361423 } from './1694432361423-AddAnalytic'
 import { AddChatHistory1694658756136 } from './1694658756136-AddChatHistory'
 import { AddAssistantEntity1699325775451 } from './1699325775451-AddAssistantEntity'
 import { AddUsedToolsToChatMessage1699481607341 } from './1699481607341-AddUsedToolsToChatMessage'
+import { AddCategoryToChatFlow1699900910291 } from './1699900910291-AddCategoryToChatFlow'
+import { AddFileAnnotationsToChatMessage1700271021237 } from './1700271021237-AddFileAnnotationsToChatMessage'
 
 export const postgresMigrations = [
     Init1693891895163,
@@ -19,5 +21,7 @@ export const postgresMigrations = [
     AddAnalytic1694432361423,
     AddChatHistory1694658756136,
     AddAssistantEntity1699325775451,
-    AddUsedToolsToChatMessage1699481607341
+    AddUsedToolsToChatMessage1699481607341,
+    AddCategoryToChatFlow1699900910291,
+    AddFileAnnotationsToChatMessage1700271021237
 ]

packages/server/src/database/migrations/sqlite/1699900910291-AddCategoryToChatFlow.ts

@@ -0,0 +1,11 @@
+import { MigrationInterface, QueryRunner } from 'typeorm'
+
+export class AddCategoryToChatFlow1699900910291 implements MigrationInterface {
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE "chat_flow" ADD COLUMN "category" TEXT;`)
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE "chat_flow" DROP COLUMN "category";`)
+    }
+}

packages/server/src/database/migrations/sqlite/1700271021237-AddFileAnnotationsToChatMessage.ts

@@ -0,0 +1,20 @@
+import { MigrationInterface, QueryRunner } from 'typeorm'
+
+export class AddFileAnnotationsToChatMessage1700271021237 implements MigrationInterface {
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(
+            `CREATE TABLE "temp_chat_message" ("id" varchar PRIMARY KEY NOT NULL, "role" varchar NOT NULL, "chatflowid" varchar NOT NULL, "content" text NOT NULL, "sourceDocuments" text, "usedTools" text, "fileAnnotations" text, "createdDate" datetime NOT NULL DEFAULT (datetime('now')), "chatType" VARCHAR NOT NULL DEFAULT 'INTERNAL', "chatId" VARCHAR NOT NULL, "memoryType" VARCHAR, "sessionId" VARCHAR);`
+        )
+        await queryRunner.query(
+            `INSERT INTO "temp_chat_message" ("id", "role", "chatflowid", "content", "sourceDocuments", "usedTools", "createdDate", "chatType", "chatId", "memoryType", "sessionId") SELECT "id", "role", "chatflowid", "content", "sourceDocuments", "usedTools", "createdDate", "chatType", "chatId", "memoryType", "sessionId" FROM "chat_message";`
+        )
+        await queryRunner.query(`DROP TABLE "chat_message";`)
+        await queryRunner.query(`ALTER TABLE "temp_chat_message" RENAME TO "chat_message";`)
+        await queryRunner.query(`CREATE INDEX "IDX_e574527322272fd838f4f0f3d3" ON "chat_message" ("chatflowid") ;`)
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`DROP TABLE IF EXISTS "temp_chat_message";`)
+        await queryRunner.query(`ALTER TABLE "chat_message" DROP COLUMN "fileAnnotations";`)
+    }
+}

packages/server/src/database/migrations/sqlite/index.ts

@@ -8,6 +8,8 @@ import { AddAnalytic1694432361423 } from './1694432361423-AddAnalytic'
 import { AddChatHistory1694657778173 } from './1694657778173-AddChatHistory'
 import { AddAssistantEntity1699325775451 } from './1699325775451-AddAssistantEntity'
 import { AddUsedToolsToChatMessage1699481607341 } from './1699481607341-AddUsedToolsToChatMessage'
+import { AddCategoryToChatFlow1699900910291 } from './1699900910291-AddCategoryToChatFlow'
+import { AddFileAnnotationsToChatMessage1700271021237 } from './1700271021237-AddFileAnnotationsToChatMessage'
 
 export const sqliteMigrations = [
     Init1693835579790,
@@ -19,5 +21,7 @@ export const sqliteMigrations = [
     AddAnalytic1694432361423,
     AddChatHistory1694657778173,
     AddAssistantEntity1699325775451,
-    AddUsedToolsToChatMessage1699481607341
+    AddUsedToolsToChatMessage1699481607341,
+    AddCategoryToChatFlow1699900910291,
+    AddFileAnnotationsToChatMessage1700271021237
 ]

packages/server/src/index.ts

@@ -138,6 +138,7 @@ export class App {
                 '/api/v1/node-icon/',
                 '/api/v1/components-credentials-icon/',
                 '/api/v1/chatflows-streaming',
+                '/api/v1/openai-assistants-file',
                 '/api/v1/ip'
             ]
             this.app.use((req, res, next) => {
@@ -355,8 +356,12 @@ export class App {
             this.AppDataSource.getRepository(ChatFlow).merge(chatflow, updateChatFlow)
             const result = await this.AppDataSource.getRepository(ChatFlow).save(chatflow)
 
-            // Update chatflowpool inSync to false, to build Langchain again because data has been changed
-            this.chatflowPool.updateInSync(chatflow.id, false)
+            // chatFlowPool is initialized only when a flow is opened
+            // if the user attempts to rename/update category without opening any flow, chatFlowPool will be undefined
+            if (this.chatflowPool) {
+                // Update chatflowpool inSync to false, to build Langchain again because data has been changed
+                this.chatflowPool.updateInSync(chatflow.id, false)
+            }
 
             return res.json(result)
         })
@@ -782,8 +787,8 @@ export class App {
 
                     await openai.beta.assistants.update(assistantDetails.id, {
                         name: assistantDetails.name,
-                        description: assistantDetails.description,
-                        instructions: assistantDetails.instructions,
+                        description: assistantDetails.description ?? '',
+                        instructions: assistantDetails.instructions ?? '',
                         model: assistantDetails.model,
                         tools: filteredTools,
                         file_ids: uniqWith(
@@ -952,7 +957,7 @@ export class App {
 
                 const results = await this.AppDataSource.getRepository(Assistant).delete({ id: req.params.id })
 
-                await openai.beta.assistants.del(assistantDetails.id)
+                if (req.query.isDeleteBoth) await openai.beta.assistants.del(assistantDetails.id)
 
                 return res.json(results)
             } catch (error: any) {
@@ -961,6 +966,14 @@ export class App {
             }
         })
 
+        // Download file from assistant
+        this.app.post('/api/v1/openai-assistants-file', async (req: Request, res: Response) => {
+            const filePath = path.join(getUserHome(), '.flowise', 'openai-assistant', req.body.fileName)
+            res.setHeader('Content-Disposition', 'attachment; filename=' + path.basename(filePath))
+            const fileStream = fs.createReadStream(filePath)
+            fileStream.pipe(res)
+        })
+
         // ----------------------------------------
         // Configuration
         // ----------------------------------------
@@ -1135,28 +1148,52 @@ export class App {
         // API Keys
         // ----------------------------------------
 
+        const addChatflowsCount = async (keys: any, res: Response) => {
+            if (keys) {
+                const updatedKeys: any[] = []
+                //iterate through keys and get chatflows
+                for (const key of keys) {
+                    const chatflows = await this.AppDataSource.getRepository(ChatFlow)
+                        .createQueryBuilder('cf')
+                        .where('cf.apikeyid = :apikeyid', { apikeyid: key.id })
+                        .getMany()
+                    const linkedChatFlows: any[] = []
+                    chatflows.map((cf) => {
+                        linkedChatFlows.push({
+                            flowName: cf.name,
+                            category: cf.category,
+                            updatedDate: cf.updatedDate
+                        })
+                    })
+                    key.chatFlows = linkedChatFlows
+                    updatedKeys.push(key)
+                }
+                return res.json(updatedKeys)
+            }
+            return res.json(keys)
+        }
         // Get api keys
         this.app.get('/api/v1/apikey', async (req: Request, res: Response) => {
             const keys = await getAPIKeys()
-            return res.json(keys)
+            return addChatflowsCount(keys, res)
         })
 
         // Add new api key
         this.app.post('/api/v1/apikey', async (req: Request, res: Response) => {
             const keys = await addAPIKey(req.body.keyName)
-            return res.json(keys)
+            return addChatflowsCount(keys, res)
         })
 
         // Update api key
         this.app.put('/api/v1/apikey/:id', async (req: Request, res: Response) => {
             const keys = await updateAPIKey(req.params.id, req.body.keyName)
-            return res.json(keys)
+            return addChatflowsCount(keys, res)
         })
 
         // Delete new api key
         this.app.delete('/api/v1/apikey/:id', async (req: Request, res: Response) => {
             const keys = await deleteAPIKey(req.params.id)
-            return res.json(keys)
+            return addChatflowsCount(keys, res)
         })
 
         // Verify api key
@@ -1499,6 +1536,7 @@ export class App {
             }
             if (result?.sourceDocuments) apiMessage.sourceDocuments = JSON.stringify(result.sourceDocuments)
             if (result?.usedTools) apiMessage.usedTools = JSON.stringify(result.usedTools)
+            if (result?.fileAnnotations) apiMessage.fileAnnotations = JSON.stringify(result.fileAnnotations)
             await this.addChatMessage(apiMessage)
 
             logger.debug(`[server]: Finished running ${nodeToExecuteData.label} (${nodeToExecuteData.id})`)

packages/server/src/utils/index.ts

@@ -842,7 +842,7 @@ export const isFlowValidForStream = (reactFlowNodes: IReactFlowNode[], endingNod
     let isValidChainOrAgent = false
     if (endingNodeData.category === 'Chains') {
         // Chains that are not available to stream
-        const blacklistChains = ['openApiChain']
+        const blacklistChains = ['openApiChain', 'vectaraQAChain']
         isValidChainOrAgent = !blacklistChains.includes(endingNodeData.name)
     } else if (endingNodeData.category === 'Agents') {
         // Agent that are available to stream
@@ -985,10 +985,14 @@ export const redactCredentialWithPasswordType = (
  * @param {any} instance
  * @param {string} chatId
  */
-export const checkMemorySessionId = (instance: any, chatId: string): string => {
+export const checkMemorySessionId = (instance: any, chatId: string): string | undefined => {
     if (instance.memory && instance.memory.isSessionIdUsingChatMessageId && chatId) {
         instance.memory.sessionId = chatId
         instance.memory.chatHistory.sessionId = chatId
     }
-    return instance.memory ? instance.memory.sessionId ?? instance.memory.chatHistory.sessionId : undefined
+
+    if (instance.memory && instance.memory.sessionId) return instance.memory.sessionId
+    else if (instance.memory && instance.memory.chatHistory && instance.memory.chatHistory.sessionId)
+        return instance.memory.chatHistory.sessionId
+    return undefined
 }