Merge branch 'main' into FEATURE/lshub-prompt

# Conflicts:
#	packages/server/src/index.ts

packages/server/src/Interface.ts

@@ -190,12 +190,6 @@ export interface IOverrideConfig {
     type: string
 }
 
-export interface IDatabaseExport {
-    chatmessages: IChatMessage[]
-    chatflows: IChatFlow[]
-    apikeys: ICommonObject[]
-}
-
 export type ICredentialDataDecrypted = ICommonObject
 
 // Plain credential object sent to server

packages/server/src/index.ts

@@ -17,7 +17,6 @@ import {
     IReactFlowNode,
     IReactFlowObject,
     INodeData,
-    IDatabaseExport,
     ICredentialReturnResponse,
     chatType,
     IChatMessage,
@@ -31,18 +30,11 @@ import {
     constructGraphs,
     resolveVariables,
     isStartNodeDependOnInput,
-    getAPIKeys,
-    addAPIKey,
-    updateAPIKey,
-    deleteAPIKey,
-    compareKeys,
     mapMimeTypeToInputField,
     findAvailableConfigs,
     isSameOverrideConfig,
-    replaceAllAPIKeys,
     isFlowValidForStream,
     databaseEntities,
-    getApiKey,
     transformToCredentialEntity,
     decryptCredentialData,
     clearAllSessionMemory,
@@ -50,7 +42,8 @@ import {
     getEncryptionKey,
     checkMemorySessionId,
     clearSessionMemoryFromViewMessageDialog,
-    getUserHome
+    getUserHome,
+    replaceChatHistory
 } from './utils'
 import { cloneDeep, omit, uniqWith, isEqual } from 'lodash'
 import { getDataSource } from './DataSource'
@@ -62,8 +55,9 @@ import { Tool } from './database/entities/Tool'
 import { Assistant } from './database/entities/Assistant'
 import { ChatflowPool } from './ChatflowPool'
 import { CachePool } from './CachePool'
-import { ICommonObject, INodeOptionsValue } from 'flowise-components'
+import { ICommonObject, IMessage, INodeOptionsValue } from 'flowise-components'
 import { createRateLimiter, getRateLimiter, initializeRateLimiter } from './utils/rateLimit'
+import { addAPIKey, compareKeys, deleteAPIKey, getApiKey, getAPIKeys, updateAPIKey } from './utils/apiKey'
 import axios from 'axios'
 import { Client } from 'langchainhub'
 import { parsePrompt } from './utils/hub'
@@ -1028,57 +1022,6 @@ export class App {
             }
         })
 
-        // ----------------------------------------
-        // Export Load Chatflow & ChatMessage & Apikeys
-        // ----------------------------------------
-
-        this.app.get('/api/v1/database/export', async (req: Request, res: Response) => {
-            const chatmessages = await this.AppDataSource.getRepository(ChatMessage).find()
-            const chatflows = await this.AppDataSource.getRepository(ChatFlow).find()
-            const apikeys = await getAPIKeys()
-            const result: IDatabaseExport = {
-                chatmessages,
-                chatflows,
-                apikeys
-            }
-            return res.json(result)
-        })
-
-        this.app.post('/api/v1/database/load', async (req: Request, res: Response) => {
-            const databaseItems: IDatabaseExport = req.body
-
-            await this.AppDataSource.getRepository(ChatFlow).delete({})
-            await this.AppDataSource.getRepository(ChatMessage).delete({})
-
-            let error = ''
-
-            // Get a new query runner instance
-            const queryRunner = this.AppDataSource.createQueryRunner()
-
-            // Start a new transaction
-            await queryRunner.startTransaction()
-
-            try {
-                const chatflows: ChatFlow[] = databaseItems.chatflows
-                const chatmessages: ChatMessage[] = databaseItems.chatmessages
-
-                await queryRunner.manager.insert(ChatFlow, chatflows)
-                await queryRunner.manager.insert(ChatMessage, chatmessages)
-
-                await queryRunner.commitTransaction()
-            } catch (err: any) {
-                error = err?.message ?? 'Error loading database'
-                await queryRunner.rollbackTransaction()
-            } finally {
-                await queryRunner.release()
-            }
-
-            await replaceAllAPIKeys(databaseItems.apikeys)
-
-            if (error) return res.status(500).send(error)
-            return res.status(201).send('OK')
-        })
-
         // ----------------------------------------
         // Upsert
         // ----------------------------------------
@@ -1377,14 +1320,14 @@ export class App {
      * @param {IReactFlowEdge[]} edges
      * @returns {string | undefined}
      */
-    findMemoryLabel(nodes: IReactFlowNode[], edges: IReactFlowEdge[]): string | undefined {
+    findMemoryLabel(nodes: IReactFlowNode[], edges: IReactFlowEdge[]): IReactFlowNode | undefined {
         const memoryNodes = nodes.filter((node) => node.data.category === 'Memory')
         const memoryNodeIds = memoryNodes.map((mem) => mem.data.id)
 
         for (const edge of edges) {
             if (memoryNodeIds.includes(edge.source)) {
                 const memoryNode = nodes.find((node) => node.data.id === edge.source)
-                return memoryNode ? memoryNode.data.label : undefined
+                return memoryNode
             }
         }
         return undefined
@@ -1505,6 +1448,19 @@ export class App {
 
                 isStreamValid = isFlowValidForStream(nodes, endingNodeData)
 
+                let chatHistory: IMessage[] | string = incomingInput.history
+                if (
+                    endingNodeData.inputs?.memory &&
+                    !incomingInput.history &&
+                    (incomingInput.chatId || incomingInput.overrideConfig?.sessionId)
+                ) {
+                    const memoryNodeId = endingNodeData.inputs?.memory.split('.')[0].replace('{{', '')
+                    const memoryNode = nodes.find((node) => node.data.id === memoryNodeId)
+                    if (memoryNode) {
+                        chatHistory = await replaceChatHistory(memoryNode, incomingInput, this.AppDataSource, databaseEntities, logger)
+                    }
+                }
+
                 /*** Get Starting Nodes with Non-Directed Graph ***/
                 const constructedObj = constructGraphs(nodes, edges, true)
                 const nonDirectedGraph = constructedObj.graph
@@ -1519,7 +1475,7 @@ export class App {
                     depthQueue,
                     this.nodesPool.componentNodes,
                     incomingInput.question,
-                    incomingInput.history,
+                    chatHistory,
                     chatId,
                     chatflowid,
                     this.AppDataSource,
@@ -1539,7 +1495,7 @@ export class App {
                     nodeToExecute.data,
                     reactFlowNodes,
                     incomingInput.question,
-                    incomingInput.history
+                    chatHistory
                 )
                 nodeToExecuteData = reactFlowNodeData
 
@@ -1556,11 +1512,17 @@ export class App {
             let sessionId = undefined
             if (nodeToExecuteData.instance) sessionId = checkMemorySessionId(nodeToExecuteData.instance, chatId)
 
-            const memoryType = this.findMemoryLabel(nodes, edges)
+            const memoryNode = this.findMemoryLabel(nodes, edges)
+            const memoryType = memoryNode?.data.label
+
+            let chatHistory: IMessage[] | string = incomingInput.history
+            if (memoryNode && !incomingInput.history && (incomingInput.chatId || incomingInput.overrideConfig?.sessionId)) {
+                chatHistory = await replaceChatHistory(memoryNode, incomingInput, this.AppDataSource, databaseEntities, logger)
+            }
 
             let result = isStreamValid
                 ? await nodeInstance.run(nodeToExecuteData, incomingInput.question, {
-                      chatHistory: incomingInput.history,
+                      chatHistory,
                       socketIO,
                       socketIOClientId: incomingInput.socketIOClientId,
                       logger,
@@ -1570,7 +1532,7 @@ export class App {
                       chatId
                   })
                 : await nodeInstance.run(nodeToExecuteData, incomingInput.question, {
-                      chatHistory: incomingInput.history,
+                      chatHistory,
                       logger,
                       appDataSource: this.AppDataSource,
                       databaseEntities,

packages/server/src/utils/apiKey.ts

@@ -0,0 +1,147 @@
+import { randomBytes, scryptSync, timingSafeEqual } from 'crypto'
+import { ICommonObject } from 'flowise-components'
+import moment from 'moment'
+import fs from 'fs'
+import path from 'path'
+import logger from './logger'
+
+/**
+ * Returns the api key path
+ * @returns {string}
+ */
+export const getAPIKeyPath = (): string => {
+    return process.env.APIKEY_PATH ? path.join(process.env.APIKEY_PATH, 'api.json') : path.join(__dirname, '..', '..', 'api.json')
+}
+
+/**
+ * Generate the api key
+ * @returns {string}
+ */
+export const generateAPIKey = (): string => {
+    const buffer = randomBytes(32)
+    return buffer.toString('base64')
+}
+
+/**
+ * Generate the secret key
+ * @param {string} apiKey
+ * @returns {string}
+ */
+export const generateSecretHash = (apiKey: string): string => {
+    const salt = randomBytes(8).toString('hex')
+    const buffer = scryptSync(apiKey, salt, 64) as Buffer
+    return `${buffer.toString('hex')}.${salt}`
+}
+
+/**
+ * Verify valid keys
+ * @param {string} storedKey
+ * @param {string} suppliedKey
+ * @returns {boolean}
+ */
+export const compareKeys = (storedKey: string, suppliedKey: string): boolean => {
+    const [hashedPassword, salt] = storedKey.split('.')
+    const buffer = scryptSync(suppliedKey, salt, 64) as Buffer
+    return timingSafeEqual(Buffer.from(hashedPassword, 'hex'), buffer)
+}
+
+/**
+ * Get API keys
+ * @returns {Promise<ICommonObject[]>}
+ */
+export const getAPIKeys = async (): Promise<ICommonObject[]> => {
+    try {
+        const content = await fs.promises.readFile(getAPIKeyPath(), 'utf8')
+        return JSON.parse(content)
+    } catch (error) {
+        const keyName = 'DefaultKey'
+        const apiKey = generateAPIKey()
+        const apiSecret = generateSecretHash(apiKey)
+        const content = [
+            {
+                keyName,
+                apiKey,
+                apiSecret,
+                createdAt: moment().format('DD-MMM-YY'),
+                id: randomBytes(16).toString('hex')
+            }
+        ]
+        await fs.promises.writeFile(getAPIKeyPath(), JSON.stringify(content), 'utf8')
+        return content
+    }
+}
+
+/**
+ * Add new API key
+ * @param {string} keyName
+ * @returns {Promise<ICommonObject[]>}
+ */
+export const addAPIKey = async (keyName: string): Promise<ICommonObject[]> => {
+    const existingAPIKeys = await getAPIKeys()
+    const apiKey = generateAPIKey()
+    const apiSecret = generateSecretHash(apiKey)
+    const content = [
+        ...existingAPIKeys,
+        {
+            keyName,
+            apiKey,
+            apiSecret,
+            createdAt: moment().format('DD-MMM-YY'),
+            id: randomBytes(16).toString('hex')
+        }
+    ]
+    await fs.promises.writeFile(getAPIKeyPath(), JSON.stringify(content), 'utf8')
+    return content
+}
+
+/**
+ * Get API Key details
+ * @param {string} apiKey
+ * @returns {Promise<ICommonObject[]>}
+ */
+export const getApiKey = async (apiKey: string) => {
+    const existingAPIKeys = await getAPIKeys()
+    const keyIndex = existingAPIKeys.findIndex((key) => key.apiKey === apiKey)
+    if (keyIndex < 0) return undefined
+    return existingAPIKeys[keyIndex]
+}
+
+/**
+ * Update existing API key
+ * @param {string} keyIdToUpdate
+ * @param {string} newKeyName
+ * @returns {Promise<ICommonObject[]>}
+ */
+export const updateAPIKey = async (keyIdToUpdate: string, newKeyName: string): Promise<ICommonObject[]> => {
+    const existingAPIKeys = await getAPIKeys()
+    const keyIndex = existingAPIKeys.findIndex((key) => key.id === keyIdToUpdate)
+    if (keyIndex < 0) return []
+    existingAPIKeys[keyIndex].keyName = newKeyName
+    await fs.promises.writeFile(getAPIKeyPath(), JSON.stringify(existingAPIKeys), 'utf8')
+    return existingAPIKeys
+}
+
+/**
+ * Delete API key
+ * @param {string} keyIdToDelete
+ * @returns {Promise<ICommonObject[]>}
+ */
+export const deleteAPIKey = async (keyIdToDelete: string): Promise<ICommonObject[]> => {
+    const existingAPIKeys = await getAPIKeys()
+    const result = existingAPIKeys.filter((key) => key.id !== keyIdToDelete)
+    await fs.promises.writeFile(getAPIKeyPath(), JSON.stringify(result), 'utf8')
+    return result
+}
+
+/**
+ * Replace all api keys
+ * @param {ICommonObject[]} content
+ * @returns {Promise<void>}
+ */
+export const replaceAllAPIKeys = async (content: ICommonObject[]): Promise<void> => {
+    try {
+        await fs.promises.writeFile(getAPIKeyPath(), JSON.stringify(content), 'utf8')
+    } catch (error) {
+        logger.error(error)
+    }
+}

packages/server/src/utils/index.ts

@@ -1,33 +1,33 @@
 import path from 'path'
 import fs from 'fs'
-import moment from 'moment'
 import logger from './logger'
 import {
+    IComponentCredentials,
     IComponentNodes,
+    ICredentialDataDecrypted,
+    ICredentialReqBody,
     IDepthQueue,
     IExploredNode,
+    INodeData,
     INodeDependencies,
     INodeDirectedGraph,
     INodeQueue,
+    IOverrideConfig,
     IReactFlowEdge,
     IReactFlowNode,
     IVariableDict,
-    INodeData,
-    IOverrideConfig,
-    ICredentialDataDecrypted,
-    IComponentCredentials,
-    ICredentialReqBody
+    IncomingInput
 } from '../Interface'
 import { cloneDeep, get, isEqual } from 'lodash'
 import {
-    ICommonObject,
+    convertChatHistoryToText,
     getInputVariables,
-    IDatabaseEntity,
     handleEscapeCharacters,
-    IMessage,
-    convertChatHistoryToText
+    ICommonObject,
+    IDatabaseEntity,
+    IMessage
 } from 'flowise-components'
-import { scryptSync, randomBytes, timingSafeEqual } from 'crypto'
+import { randomBytes } from 'crypto'
 import { AES, enc } from 'crypto-js'
 
 import { ChatFlow } from '../database/entities/ChatFlow'
@@ -217,7 +217,7 @@ export const buildLangchain = async (
     depthQueue: IDepthQueue,
     componentNodes: IComponentNodes,
     question: string,
-    chatHistory: IMessage[],
+    chatHistory: IMessage[] | string,
     chatId: string,
     chatflowid: string,
     appDataSource: DataSource,
@@ -348,8 +348,8 @@ export const clearAllSessionMemory = async (
             node.data.inputs.sessionId = sessionId
         }
 
-        if (newNodeInstance.clearSessionMemory) {
-            await newNodeInstance?.clearSessionMemory(node.data, { chatId, appDataSource, databaseEntities, logger })
+        if (newNodeInstance.memoryMethods && newNodeInstance.memoryMethods.clearSessionMemory) {
+            await newNodeInstance.memoryMethods.clearSessionMemory(node.data, { chatId, appDataSource, databaseEntities, logger })
         }
     }
 }
@@ -381,8 +381,8 @@ export const clearSessionMemoryFromViewMessageDialog = async (
 
         if (sessionId && node.data.inputs) node.data.inputs.sessionId = sessionId
 
-        if (newNodeInstance.clearSessionMemory) {
-            await newNodeInstance?.clearSessionMemory(node.data, { chatId, appDataSource, databaseEntities, logger })
+        if (newNodeInstance.memoryMethods && newNodeInstance.memoryMethods.clearSessionMemory) {
+            await newNodeInstance.memoryMethods.clearSessionMemory(node.data, { chatId, appDataSource, databaseEntities, logger })
             return
         }
     }
@@ -400,7 +400,7 @@ export const getVariableValue = (
     paramValue: string,
     reactFlowNodes: IReactFlowNode[],
     question: string,
-    chatHistory: IMessage[],
+    chatHistory: IMessage[] | string,
     isAcceptVariable = false
 ) => {
     let returnVal = paramValue
@@ -433,7 +433,10 @@ export const getVariableValue = (
             }
 
             if (isAcceptVariable && variableFullPath === CHAT_HISTORY_VAR_PREFIX) {
-                variableDict[`{{${variableFullPath}}}`] = handleEscapeCharacters(convertChatHistoryToText(chatHistory), false)
+                variableDict[`{{${variableFullPath}}}`] = handleEscapeCharacters(
+                    typeof chatHistory === 'string' ? chatHistory : convertChatHistoryToText(chatHistory),
+                    false
+                )
             }
 
             // Split by first occurrence of '.' to get just nodeId
@@ -476,7 +479,7 @@ export const resolveVariables = (
     reactFlowNodeData: INodeData,
     reactFlowNodes: IReactFlowNode[],
     question: string,
-    chatHistory: IMessage[]
+    chatHistory: IMessage[] | string
 ): INodeData => {
     let flowNodeData = cloneDeep(reactFlowNodeData)
     const types = 'inputs'
@@ -593,147 +596,6 @@ export const isSameOverrideConfig = (
     return false
 }
 
-/**
- * Returns the api key path
- * @returns {string}
- */
-export const getAPIKeyPath = (): string => {
-    return process.env.APIKEY_PATH ? path.join(process.env.APIKEY_PATH, 'api.json') : path.join(__dirname, '..', '..', 'api.json')
-}
-
-/**
- * Generate the api key
- * @returns {string}
- */
-export const generateAPIKey = (): string => {
-    const buffer = randomBytes(32)
-    return buffer.toString('base64')
-}
-
-/**
- * Generate the secret key
- * @param {string} apiKey
- * @returns {string}
- */
-export const generateSecretHash = (apiKey: string): string => {
-    const salt = randomBytes(8).toString('hex')
-    const buffer = scryptSync(apiKey, salt, 64) as Buffer
-    return `${buffer.toString('hex')}.${salt}`
-}
-
-/**
- * Verify valid keys
- * @param {string} storedKey
- * @param {string} suppliedKey
- * @returns {boolean}
- */
-export const compareKeys = (storedKey: string, suppliedKey: string): boolean => {
-    const [hashedPassword, salt] = storedKey.split('.')
-    const buffer = scryptSync(suppliedKey, salt, 64) as Buffer
-    return timingSafeEqual(Buffer.from(hashedPassword, 'hex'), buffer)
-}
-
-/**
- * Get API keys
- * @returns {Promise<ICommonObject[]>}
- */
-export const getAPIKeys = async (): Promise<ICommonObject[]> => {
-    try {
-        const content = await fs.promises.readFile(getAPIKeyPath(), 'utf8')
-        return JSON.parse(content)
-    } catch (error) {
-        const keyName = 'DefaultKey'
-        const apiKey = generateAPIKey()
-        const apiSecret = generateSecretHash(apiKey)
-        const content = [
-            {
-                keyName,
-                apiKey,
-                apiSecret,
-                createdAt: moment().format('DD-MMM-YY'),
-                id: randomBytes(16).toString('hex')
-            }
-        ]
-        await fs.promises.writeFile(getAPIKeyPath(), JSON.stringify(content), 'utf8')
-        return content
-    }
-}
-
-/**
- * Add new API key
- * @param {string} keyName
- * @returns {Promise<ICommonObject[]>}
- */
-export const addAPIKey = async (keyName: string): Promise<ICommonObject[]> => {
-    const existingAPIKeys = await getAPIKeys()
-    const apiKey = generateAPIKey()
-    const apiSecret = generateSecretHash(apiKey)
-    const content = [
-        ...existingAPIKeys,
-        {
-            keyName,
-            apiKey,
-            apiSecret,
-            createdAt: moment().format('DD-MMM-YY'),
-            id: randomBytes(16).toString('hex')
-        }
-    ]
-    await fs.promises.writeFile(getAPIKeyPath(), JSON.stringify(content), 'utf8')
-    return content
-}
-
-/**
- * Get API Key details
- * @param {string} apiKey
- * @returns {Promise<ICommonObject[]>}
- */
-export const getApiKey = async (apiKey: string) => {
-    const existingAPIKeys = await getAPIKeys()
-    const keyIndex = existingAPIKeys.findIndex((key) => key.apiKey === apiKey)
-    if (keyIndex < 0) return undefined
-    return existingAPIKeys[keyIndex]
-}
-
-/**
- * Update existing API key
- * @param {string} keyIdToUpdate
- * @param {string} newKeyName
- * @returns {Promise<ICommonObject[]>}
- */
-export const updateAPIKey = async (keyIdToUpdate: string, newKeyName: string): Promise<ICommonObject[]> => {
-    const existingAPIKeys = await getAPIKeys()
-    const keyIndex = existingAPIKeys.findIndex((key) => key.id === keyIdToUpdate)
-    if (keyIndex < 0) return []
-    existingAPIKeys[keyIndex].keyName = newKeyName
-    await fs.promises.writeFile(getAPIKeyPath(), JSON.stringify(existingAPIKeys), 'utf8')
-    return existingAPIKeys
-}
-
-/**
- * Delete API key
- * @param {string} keyIdToDelete
- * @returns {Promise<ICommonObject[]>}
- */
-export const deleteAPIKey = async (keyIdToDelete: string): Promise<ICommonObject[]> => {
-    const existingAPIKeys = await getAPIKeys()
-    const result = existingAPIKeys.filter((key) => key.id !== keyIdToDelete)
-    await fs.promises.writeFile(getAPIKeyPath(), JSON.stringify(result), 'utf8')
-    return result
-}
-
-/**
- * Replace all api keys
- * @param {ICommonObject[]} content
- * @returns {Promise<void>}
- */
-export const replaceAllAPIKeys = async (content: ICommonObject[]): Promise<void> => {
-    try {
-        await fs.promises.writeFile(getAPIKeyPath(), JSON.stringify(content), 'utf8')
-    } catch (error) {
-        logger.error(error)
-    }
-}
-
 /**
  * Map MimeType to InputField
  * @param {string} mimeType
@@ -1015,3 +877,39 @@ export const checkMemorySessionId = (instance: any, chatId: string): string | un
         return instance.memory.chatHistory.sessionId
     return undefined
 }
+
+/**
+ * Replace chatHistory if incomingInput.history is empty and sessionId/chatId is provided
+ * @param {IReactFlowNode} memoryNode
+ * @param {IncomingInput} incomingInput
+ * @param {DataSource} appDataSource
+ * @param {IDatabaseEntity} databaseEntities
+ * @param {any} logger
+ * @returns {string}
+ */
+export const replaceChatHistory = async (
+    memoryNode: IReactFlowNode,
+    incomingInput: IncomingInput,
+    appDataSource: DataSource,
+    databaseEntities: IDatabaseEntity,
+    logger: any
+): Promise<string> => {
+    const nodeInstanceFilePath = memoryNode.data.filePath as string
+    const nodeModule = await import(nodeInstanceFilePath)
+    const newNodeInstance = new nodeModule.nodeClass()
+
+    if (incomingInput.overrideConfig?.sessionId && memoryNode.data.inputs) {
+        memoryNode.data.inputs.sessionId = incomingInput.overrideConfig.sessionId
+    }
+
+    if (newNodeInstance.memoryMethods && newNodeInstance.memoryMethods.getChatMessages) {
+        return await newNodeInstance.memoryMethods.getChatMessages(memoryNode.data, {
+            chatId: incomingInput.chatId,
+            appDataSource,
+            databaseEntities,
+            logger
+        })
+    }
+
+    return ''
+}