Merge branch 'main' into feature/output-parsers

# Conflicts:
#	packages/ui/src/views/chatmessage/ChatMessage.js

packages/components/credentials/RedisCacheUrlApi.credential.ts

@@ -0,0 +1,25 @@
+import { INodeParams, INodeCredential } from '../src/Interface'
+
+class RedisCacheUrlApi implements INodeCredential {
+    label: string
+    name: string
+    version: number
+    description: string
+    inputs: INodeParams[]
+
+    constructor() {
+        this.label = 'Redis Cache URL'
+        this.name = 'redisCacheUrlApi'
+        this.version = 1.0
+        this.inputs = [
+            {
+                label: 'Redis URL',
+                name: 'redisUrl',
+                type: 'string',
+                default: '127.0.0.1'
+            }
+        ]
+    }
+}
+
+module.exports = { credClass: RedisCacheUrlApi }

packages/components/credentials/SearchApi.credential.ts

@@ -0,0 +1,26 @@
+import { INodeParams, INodeCredential } from '../src/Interface'
+
+class SearchApi implements INodeCredential {
+    label: string
+    name: string
+    version: number
+    description: string
+    inputs: INodeParams[]
+
+    constructor() {
+        this.label = 'Search API'
+        this.name = 'searchApi'
+        this.version = 1.0
+        this.description =
+            'Sign in to <a target="_blank" href="https://www.searchapi.io/">SearchApi</a> to obtain a free API key from the dashboard.'
+        this.inputs = [
+            {
+                label: 'SearchApi API Key',
+                name: 'searchApiKey',
+                type: 'password'
+            }
+        ]
+    }
+}
+
+module.exports = { credClass: SearchApi }

packages/components/nodes/agents/CSVAgent/CSVAgent.ts

@@ -131,7 +131,7 @@ json.dumps(my_dict)`
                 const code = `import pandas as pd\n${pythonCode}`
                 finalResult = await pyodide.runPythonAsync(code)
             } catch (error) {
-                throw new Error(`Sorry, I'm unable to find answer for question: "${input}" using follwoing code: "${pythonCode}"`)
+                throw new Error(`Sorry, I'm unable to find answer for question: "${input}" using following code: "${pythonCode}"`)
             }
         }
 

packages/components/nodes/cache/RedisCache/RedisCache.ts

@@ -30,7 +30,7 @@ class RedisCache implements INode {
             name: 'credential',
             type: 'credential',
             optional: true,
-            credentialNames: ['redisCacheApi']
+            credentialNames: ['redisCacheApi', 'redisCacheUrlApi']
         }
         this.inputs = [
             {
@@ -48,17 +48,24 @@ class RedisCache implements INode {
         const ttl = nodeData.inputs?.ttl as string
 
         const credentialData = await getCredentialData(nodeData.credential ?? '', options)
-        const username = getCredentialParam('redisCacheUser', credentialData, nodeData)
+        const redisUrl = getCredentialParam('redisUrl', credentialData, nodeData)
-        const password = getCredentialParam('redisCachePwd', credentialData, nodeData)
-        const portStr = getCredentialParam('redisCachePort', credentialData, nodeData)
-        const host = getCredentialParam('redisCacheHost', credentialData, nodeData)
 
-        const client = new Redis({
+        let client: Redis
-            port: portStr ? parseInt(portStr) : 6379,
+        if (!redisUrl || redisUrl === '') {
-            host,
+            const username = getCredentialParam('redisCacheUser', credentialData, nodeData)
-            username,
+            const password = getCredentialParam('redisCachePwd', credentialData, nodeData)
-            password
+            const portStr = getCredentialParam('redisCachePort', credentialData, nodeData)
-        })
+            const host = getCredentialParam('redisCacheHost', credentialData, nodeData)
+
+            client = new Redis({
+                port: portStr ? parseInt(portStr) : 6379,
+                host,
+                username,
+                password
+            })
+        } else {
+            client = new Redis(redisUrl)
+        }
 
         const redisClient = new LangchainRedisCache(client)
 

packages/components/nodes/cache/RedisCache/RedisEmbeddingsCache.ts

@@ -30,7 +30,7 @@ class RedisEmbeddingsCache implements INode {
             name: 'credential',
             type: 'credential',
             optional: true,
-            credentialNames: ['redisCacheApi']
+            credentialNames: ['redisCacheApi', 'redisCacheUrlApi']
         }
         this.inputs = [
             {
@@ -63,17 +63,25 @@ class RedisEmbeddingsCache implements INode {
         const underlyingEmbeddings = nodeData.inputs?.embeddings as Embeddings
 
         const credentialData = await getCredentialData(nodeData.credential ?? '', options)
-        const username = getCredentialParam('redisCacheUser', credentialData, nodeData)
+        const redisUrl = getCredentialParam('redisUrl', credentialData, nodeData)
-        const password = getCredentialParam('redisCachePwd', credentialData, nodeData)
+
-        const portStr = getCredentialParam('redisCachePort', credentialData, nodeData)
+        let client: Redis
-        const host = getCredentialParam('redisCacheHost', credentialData, nodeData)
+        if (!redisUrl || redisUrl === '') {
+            const username = getCredentialParam('redisCacheUser', credentialData, nodeData)
+            const password = getCredentialParam('redisCachePwd', credentialData, nodeData)
+            const portStr = getCredentialParam('redisCachePort', credentialData, nodeData)
+            const host = getCredentialParam('redisCacheHost', credentialData, nodeData)
+
+            client = new Redis({
+                port: portStr ? parseInt(portStr) : 6379,
+                host,
+                username,
+                password
+            })
+        } else {
+            client = new Redis(redisUrl)
+        }
 
-        const client = new Redis({
-            port: portStr ? parseInt(portStr) : 6379,
-            host,
-            username,
-            password
-        })
         ttl ??= '3600'
         let ttlNumber = parseInt(ttl, 10)
         const redisStore = new RedisByteStore({

packages/components/nodes/chains/ConversationChain/ConversationChain.ts

@@ -71,7 +71,9 @@ class ConversationChain_Chains implements INode {
         const flattenDocs = docs && docs.length ? flatten(docs) : []
         const finalDocs = []
         for (let i = 0; i < flattenDocs.length; i += 1) {
-            finalDocs.push(new Document(flattenDocs[i]))
+            if (flattenDocs[i] && flattenDocs[i].pageContent) {
+                finalDocs.push(new Document(flattenDocs[i]))
+            }
         }
 
         let finalText = ''

packages/components/nodes/documentloaders/SearchApi/SearchAPI.ts

@@ -0,0 +1,109 @@
+import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
+import { TextSplitter } from 'langchain/text_splitter'
+import { SearchApiLoader } from 'langchain/document_loaders/web/searchapi'
+import { getCredentialData, getCredentialParam } from '../../../src'
+
+// Provides access to multiple search engines using the SearchApi.
+// For available parameters & engines, refer to: https://www.searchapi.io/docs/google
+class SearchAPI_DocumentLoaders implements INode {
+    label: string
+    name: string
+    version: number
+    description: string
+    type: string
+    icon: string
+    category: string
+    baseClasses: string[]
+    credential: INodeParams
+    inputs: INodeParams[]
+
+    constructor() {
+        this.label = 'SearchApi For Web Search'
+        this.name = 'searchApi'
+        this.version = 1.0
+        this.type = 'Document'
+        this.icon = 'searchapi.svg'
+        this.category = 'Document Loaders'
+        this.description = 'Load data from real-time search results'
+        this.baseClasses = [this.type]
+        this.credential = {
+            label: 'Connect Credential',
+            name: 'credential',
+            type: 'credential',
+            optional: false,
+            credentialNames: ['searchApi']
+        }
+        this.inputs = [
+            {
+                label: 'Query',
+                name: 'query',
+                type: 'string',
+                optional: true
+            },
+            {
+                label: 'Custom Parameters',
+                name: 'customParameters',
+                type: 'json',
+                optional: true,
+                additionalParams: true
+            },
+            {
+                label: 'Text Splitter',
+                name: 'textSplitter',
+                type: 'TextSplitter',
+                optional: true
+            },
+            {
+                label: 'Metadata',
+                name: 'metadata',
+                type: 'json',
+                optional: true,
+                additionalParams: true
+            }
+        ]
+    }
+
+    async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
+        const textSplitter = nodeData.inputs?.textSplitter as TextSplitter
+        const query = nodeData.inputs?.query as string
+        const customParameters = nodeData.inputs?.customParameters
+        const metadata = nodeData.inputs?.metadata
+
+        // Fetch the API credentials for this node
+        const credentialData = await getCredentialData(nodeData.credential ?? '', options)
+        const searchApiKey = getCredentialParam('searchApiKey', credentialData, nodeData)
+
+        // Check and parse custom parameters (should be JSON or object)
+        const parsedParameters = typeof customParameters === 'object' ? customParameters : JSON.parse(customParameters || '{}')
+
+        // Prepare the configuration for the SearchApiLoader
+        const loaderConfig = {
+            q: query,
+            apiKey: searchApiKey,
+            ...parsedParameters
+        }
+
+        // Initialize the loader with the given configuration
+        const loader = new SearchApiLoader(loaderConfig)
+
+        // Fetch documents, split if a text splitter is provided
+        const docs = textSplitter ? await loader.loadAndSplit() : await loader.load()
+
+        if (metadata) {
+            const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata)
+            return docs.map((doc) => {
+                return {
+                    ...doc,
+                    metadata: {
+                        ...doc.metadata,
+                        ...parsedMetadata
+                    }
+                }
+            })
+        }
+
+        return docs
+    }
+}
+
+module.exports = { nodeClass: SearchAPI_DocumentLoaders }

packages/components/nodes/documentloaders/SearchApi/searchapi.svg

@@ -0,0 +1 @@
+<svg id="SvgjsSvg1001" width="75.27131652832031" height="63.92396926879883" xmlns="http://www.w3.org/2000/svg" version="1.1" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:svgjs="http://svgjs.com/svgjs" viewBox="0 0 75.27131652832031 63.92396926879883"><defs id="SvgjsDefs1002"></defs><rect id="SvgjsRect1008" width="75.27131652832031" height="63.92396926879883" fill="transparent"></rect><g id="SvgjsG1009" transform="matrix(1,0,0,1,-39.50003433227539,-50.53549575805664)"><title>0479_octopus_verti</title><path id="color_1" d="M97.24234,109.8245a2.57759,2.57759,0,0,1-2.57778,2.57778,9.80672,9.80672,0,0,1-9.79558-9.79557V91.58366a2.57779,2.57779,0,0,1,5.15557,0v11.02305a4.64509,4.64509,0,0,0,4.64,4.64A2.57759,2.57759,0,0,1,97.24234,109.8245ZM112.19348,93.223a2.57759,2.57759,0,0,0-2.57778,2.57779,4.64,4.64,0,1,1-9.28,0V73.73554a23.2,23.2,0,1,0-46.40009,0V95.80076a4.64,4.64,0,1,1-9.28,0,2.57779,2.57779,0,1,0-5.15557,0,9.79558,9.79558,0,0,0,19.59115,0V73.73554a18.04449,18.04449,0,0,1,36.089,0V95.80076a9.79558,9.79558,0,0,0,19.59115,0A2.57759,2.57759,0,0,0,112.19348,93.223ZM77.13563,91.78a2.57759,2.57759,0,0,0-2.57778,2.57778v17.52893a2.57779,2.57779,0,0,0,5.15557,0V94.3578A2.57759,2.57759,0,0,0,77.13563,91.78ZM66.8245,89.00588a2.57759,2.57759,0,0,0-2.57778,2.57778v11.02305a4.64509,4.64509,0,0,1-4.64,4.64,2.57778,2.57778,0,1,0,0,5.15556,9.80671,9.80671,0,0,0,9.79557-9.79557V91.58366A2.57759,2.57759,0,0,0,66.8245,89.00588ZM69.918,70.12639a3.6089,3.6089,0,1,0,3.6089,3.6089A3.60891,3.60891,0,0,0,69.918,70.12639Zm18.04448,3.6089a3.6089,3.6089,0,1,0-3.60889,3.60889A3.60891,3.60891,0,0,0,87.96251,73.73529Z" fill="#3730a3"></path></g></svg>

packages/components/nodes/documentloaders/Text/Text.ts

@@ -19,7 +19,7 @@ class Text_DocumentLoaders implements INode {
     constructor() {
         this.label = 'Text File'
         this.name = 'textFile'
-        this.version = 2.0
+        this.version = 3.0
         this.type = 'Document'
         this.icon = 'textFile.svg'
         this.category = 'Document Loaders'
@@ -30,7 +30,8 @@ class Text_DocumentLoaders implements INode {
                 label: 'Txt File',
                 name: 'txtFile',
                 type: 'file',
-                fileType: '.txt'
+                fileType:
+                    '.txt, .html, .aspx, .asp, .cpp, .c, .cs, .css, .go, .h, .java, .js, .less, .ts, .php, .proto, .python, .py, .rst, .ruby, .rb, .rs, .scala, .sc, .scss, .sol, .sql, .swift, .markdown, .md, .tex, .ltx, .vb, .xml'
             },
             {
                 label: 'Text Splitter',

packages/components/nodes/documentloaders/VectorStoreToDocument/VectorStoreToDocument.ts

@@ -17,7 +17,7 @@ class VectorStoreToDocument_DocumentLoaders implements INode {
     constructor() {
         this.label = 'VectorStore To Document'
         this.name = 'vectorStoreToDocument'
-        this.version = 1.0
+        this.version = 2.0
         this.type = 'Document'
         this.icon = 'vectorretriever.svg'
         this.category = 'Document Loaders'
@@ -29,6 +29,14 @@ class VectorStoreToDocument_DocumentLoaders implements INode {
                 name: 'vectorStore',
                 type: 'VectorStore'
             },
+            {
+                label: 'Query',
+                name: 'query',
+                type: 'string',
+                description: 'Query to retrieve documents from vector database. If not specified, user question will be used',
+                optional: true,
+                acceptVariable: true
+            },
             {
                 label: 'Minimum Score (%)',
                 name: 'minScore',
@@ -56,11 +64,12 @@ class VectorStoreToDocument_DocumentLoaders implements INode {
     async init(nodeData: INodeData, input: string): Promise<any> {
         const vectorStore = nodeData.inputs?.vectorStore as VectorStore
         const minScore = nodeData.inputs?.minScore as number
+        const query = nodeData.inputs?.query as string
         const output = nodeData.outputs?.output as string
 
         const topK = (vectorStore as any)?.k ?? 4
 
-        const docs = await vectorStore.similaritySearchWithScore(input, topK)
+        const docs = await vectorStore.similaritySearchWithScore(query ?? input, topK)
         // eslint-disable-next-line no-console
         console.log('\x1b[94m\x1b[1m\n*****VectorStore Documents*****\n\x1b[0m\x1b[0m')
         // eslint-disable-next-line no-console

packages/components/nodes/embeddings/AWSBedrockEmbedding/AWSBedrockEmbedding.ts

@@ -79,7 +79,10 @@ class AWSBedrockEmbedding_Embeddings implements INode {
                 label: 'Model Name',
                 name: 'model',
                 type: 'options',
-                options: [{ label: 'amazon.titan-embed-text-v1', name: 'amazon.titan-embed-text-v1' }],
+                options: [
+                    { label: 'amazon.titan-embed-text-v1', name: 'amazon.titan-embed-text-v1' },
+                    { label: 'amazon.titan-embed-g1-text-02', name: 'amazon.titan-embed-g1-text-02' }
+                ],
                 default: 'amazon.titan-embed-text-v1'
             }
         ]

packages/components/nodes/embeddings/OpenAIEmbeddingCustom/OpenAIEmbeddingCustom.ts

@@ -0,0 +1,94 @@
+import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
+import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
+import { OpenAIEmbeddings, OpenAIEmbeddingsParams } from 'langchain/embeddings/openai'
+
+class OpenAIEmbeddingCustom_Embeddings implements INode {
+    label: string
+    name: string
+    version: number
+    type: string
+    icon: string
+    category: string
+    description: string
+    baseClasses: string[]
+    credential: INodeParams
+    inputs: INodeParams[]
+
+    constructor() {
+        this.label = 'OpenAI Embeddings Custom'
+        this.name = 'openAIEmbeddingsCustom'
+        this.version = 1.0
+        this.type = 'OpenAIEmbeddingsCustom'
+        this.icon = 'openai.png'
+        this.category = 'Embeddings'
+        this.description = 'OpenAI API to generate embeddings for a given text'
+        this.baseClasses = [this.type, ...getBaseClasses(OpenAIEmbeddings)]
+        this.credential = {
+            label: 'Connect Credential',
+            name: 'credential',
+            type: 'credential',
+            credentialNames: ['openAIApi']
+        }
+        this.inputs = [
+            {
+                label: 'Strip New Lines',
+                name: 'stripNewLines',
+                type: 'boolean',
+                optional: true,
+                additionalParams: true
+            },
+            {
+                label: 'Batch Size',
+                name: 'batchSize',
+                type: 'number',
+                optional: true,
+                additionalParams: true
+            },
+            {
+                label: 'Timeout',
+                name: 'timeout',
+                type: 'number',
+                optional: true,
+                additionalParams: true
+            },
+            {
+                label: 'BasePath',
+                name: 'basepath',
+                type: 'string',
+                optional: true,
+                additionalParams: true
+            },
+            {
+                label: 'Model Name',
+                name: 'modelName',
+                type: 'string',
+                optional: true
+            }
+        ]
+    }
+
+    async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
+        const stripNewLines = nodeData.inputs?.stripNewLines as boolean
+        const batchSize = nodeData.inputs?.batchSize as string
+        const timeout = nodeData.inputs?.timeout as string
+        const basePath = nodeData.inputs?.basepath as string
+        const modelName = nodeData.inputs?.modelName as string
+
+        const credentialData = await getCredentialData(nodeData.credential ?? '', options)
+        const openAIApiKey = getCredentialParam('openAIApiKey', credentialData, nodeData)
+
+        const obj: Partial<OpenAIEmbeddingsParams> & { openAIApiKey?: string } = {
+            openAIApiKey
+        }
+
+        if (stripNewLines) obj.stripNewLines = stripNewLines
+        if (batchSize) obj.batchSize = parseInt(batchSize, 10)
+        if (timeout) obj.timeout = parseInt(timeout, 10)
+        if (modelName) obj.modelName = modelName
+
+        const model = new OpenAIEmbeddings(obj, { basePath })
+        return model
+    }
+}
+
+module.exports = { nodeClass: OpenAIEmbeddingCustom_Embeddings }

packages/components/nodes/memory/BufferWindowMemory/BufferWindowMemory.ts

@@ -20,7 +20,7 @@ class BufferWindowMemory_Memory implements INode {
         this.type = 'BufferWindowMemory'
         this.icon = 'memory.svg'
         this.category = 'Memory'
-        this.description = 'Uses a window of size k to surface the last k back-and-forths to use as memory'
+        this.description = 'Uses a window of size k to surface the last k back-and-forth to use as memory'
         this.baseClasses = [this.type, ...getBaseClasses(BufferWindowMemory)]
         this.inputs = [
             {
@@ -40,7 +40,7 @@ class BufferWindowMemory_Memory implements INode {
                 name: 'k',
                 type: 'number',
                 default: '4',
-                description: 'Window of size k to surface the last k back-and-forths to use as memory.'
+                description: 'Window of size k to surface the last k back-and-forth to use as memory.'
             }
         ]
     }

packages/components/nodes/memory/RedisBackedChatMemory/RedisBackedChatMemory.ts

@@ -1,9 +1,9 @@
-import { INode, INodeData, INodeParams } from '../../../src/Interface'
+import { INode, INodeData, INodeParams, ICommonObject } from '../../../src/Interface'
-import { getBaseClasses } from '../../../src/utils'
+import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
-import { ICommonObject } from '../../../src'
 import { BufferMemory, BufferMemoryInput } from 'langchain/memory'
-import { RedisChatMessageHistory, RedisChatMessageHistoryInput } from 'langchain/stores/message/redis'
+import { RedisChatMessageHistory, RedisChatMessageHistoryInput } from 'langchain/stores/message/ioredis'
-import { createClient } from 'redis'
+import { mapStoredMessageToChatMessage, BaseMessage } from 'langchain/schema'
+import { Redis } from 'ioredis'
 
 class RedisBackedChatMemory_Memory implements INode {
     label: string
@@ -15,23 +15,25 @@ class RedisBackedChatMemory_Memory implements INode {
     category: string
     baseClasses: string[]
     inputs: INodeParams[]
+    credential: INodeParams
 
     constructor() {
         this.label = 'Redis-Backed Chat Memory'
         this.name = 'RedisBackedChatMemory'
-        this.version = 1.0
+        this.version = 2.0
         this.type = 'RedisBackedChatMemory'
         this.icon = 'redis.svg'
         this.category = 'Memory'
         this.description = 'Summarizes the conversation and stores the memory in Redis server'
         this.baseClasses = [this.type, ...getBaseClasses(BufferMemory)]
+        this.credential = {
+            label: 'Connect Credential',
+            name: 'credential',
+            type: 'credential',
+            optional: true,
+            credentialNames: ['redisCacheApi', 'redisCacheUrlApi']
+        }
         this.inputs = [
-            {
-                label: 'Base URL',
-                name: 'baseURL',
-                type: 'string',
-                default: 'redis://localhost:6379'
-            },
             {
                 label: 'Session Id',
                 name: 'sessionId',
@@ -60,11 +62,11 @@ class RedisBackedChatMemory_Memory implements INode {
     }
 
     async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
-        return initalizeRedis(nodeData, options)
+        return await initalizeRedis(nodeData, options)
     }
 
     async clearSessionMemory(nodeData: INodeData, options: ICommonObject): Promise<void> {
-        const redis = initalizeRedis(nodeData, options)
+        const redis = await initalizeRedis(nodeData, options)
         const sessionId = nodeData.inputs?.sessionId as string
         const chatId = options?.chatId as string
         options.logger.info(`Clearing Redis memory session ${sessionId ? sessionId : chatId}`)
@@ -73,8 +75,7 @@ class RedisBackedChatMemory_Memory implements INode {
     }
 }
 
-const initalizeRedis = (nodeData: INodeData, options: ICommonObject): BufferMemory => {
+const initalizeRedis = async (nodeData: INodeData, options: ICommonObject): Promise<BufferMemory> => {
-    const baseURL = nodeData.inputs?.baseURL as string
     const sessionId = nodeData.inputs?.sessionId as string
     const sessionTTL = nodeData.inputs?.sessionTTL as number
     const memoryKey = nodeData.inputs?.memoryKey as string
@@ -83,10 +84,29 @@ const initalizeRedis = (nodeData: INodeData, options: ICommonObject): BufferMemo
     let isSessionIdUsingChatMessageId = false
     if (!sessionId && chatId) isSessionIdUsingChatMessageId = true
 
-    const redisClient = createClient({ url: baseURL })
+    const credentialData = await getCredentialData(nodeData.credential ?? '', options)
+    const redisUrl = getCredentialParam('redisUrl', credentialData, nodeData)
+
+    let client: Redis
+    if (!redisUrl || redisUrl === '') {
+        const username = getCredentialParam('redisCacheUser', credentialData, nodeData)
+        const password = getCredentialParam('redisCachePwd', credentialData, nodeData)
+        const portStr = getCredentialParam('redisCachePort', credentialData, nodeData)
+        const host = getCredentialParam('redisCacheHost', credentialData, nodeData)
+
+        client = new Redis({
+            port: portStr ? parseInt(portStr) : 6379,
+            host,
+            username,
+            password
+        })
+    } else {
+        client = new Redis(redisUrl)
+    }
+
     let obj: RedisChatMessageHistoryInput = {
         sessionId: sessionId ? sessionId : chatId,
-        client: redisClient
+        client
     }
 
     if (sessionTTL) {
@@ -98,10 +118,27 @@ const initalizeRedis = (nodeData: INodeData, options: ICommonObject): BufferMemo
 
     const redisChatMessageHistory = new RedisChatMessageHistory(obj)
 
+    redisChatMessageHistory.getMessages = async (): Promise<BaseMessage[]> => {
+        const rawStoredMessages = await client.lrange((redisChatMessageHistory as any).sessionId, 0, -1)
+        const orderedMessages = rawStoredMessages.reverse().map((message) => JSON.parse(message))
+        return orderedMessages.map(mapStoredMessageToChatMessage)
+    }
+
+    redisChatMessageHistory.addMessage = async (message: BaseMessage): Promise<void> => {
+        const messageToAdd = [message].map((msg) => msg.toDict())
+        await client.lpush((redisChatMessageHistory as any).sessionId, JSON.stringify(messageToAdd[0]))
+        if (sessionTTL) {
+            await client.expire((redisChatMessageHistory as any).sessionId, sessionTTL)
+        }
+    }
+
+    redisChatMessageHistory.clear = async (): Promise<void> => {
+        await client.del((redisChatMessageHistory as any).sessionId)
+    }
+
     const memory = new BufferMemoryExtended({
         memoryKey,
         chatHistory: redisChatMessageHistory,
-        returnMessages: true,
         isSessionIdUsingChatMessageId
     })
     return memory

packages/components/nodes/memory/ZepMemory/ZepMemory.ts

@@ -60,7 +60,7 @@ class ZepMemory_Memory implements INode {
                 name: 'k',
                 type: 'number',
                 default: '10',
-                description: 'Window of size k to surface the last k back-and-forths to use as memory.'
+                description: 'Window of size k to surface the last k back-and-forth to use as memory.'
             },
             {
                 label: 'Auto Summary Template',

packages/components/nodes/prompts/ChatPromptTemplate/ChatPromptTemplate.ts

@@ -51,7 +51,7 @@ class ChatPromptTemplate_Prompts implements INode {
     async init(nodeData: INodeData): Promise<any> {
         const systemMessagePrompt = nodeData.inputs?.systemMessagePrompt as string
         const humanMessagePrompt = nodeData.inputs?.humanMessagePrompt as string
-        const promptValuesStr = nodeData.inputs?.promptValues as string
+        const promptValuesStr = nodeData.inputs?.promptValues
 
         const prompt = ChatPromptTemplate.fromMessages([
             SystemMessagePromptTemplate.fromTemplate(systemMessagePrompt),
@@ -60,7 +60,11 @@ class ChatPromptTemplate_Prompts implements INode {
 
         let promptValues: ICommonObject = {}
         if (promptValuesStr) {
-            promptValues = JSON.parse(promptValuesStr)
+            try {
+                promptValues = typeof promptValuesStr === 'object' ? promptValuesStr : JSON.parse(promptValuesStr)
+            } catch (exception) {
+                throw new Error("Invalid JSON in the ChatPromptTemplate's promptValues: " + exception)
+            }
         }
         // @ts-ignore
         prompt.promptValues = promptValues

packages/components/nodes/prompts/FewShotPromptTemplate/FewShotPromptTemplate.ts

@@ -55,7 +55,7 @@ class FewShotPromptTemplate_Prompts implements INode {
                 placeholder: `Word: {input}\nAntonym:`
             },
             {
-                label: 'Example Seperator',
+                label: 'Example Separator',
                 name: 'exampleSeparator',
                 type: 'string',
                 placeholder: `\n\n`
@@ -80,7 +80,7 @@ class FewShotPromptTemplate_Prompts implements INode {
     }
 
     async init(nodeData: INodeData): Promise<any> {
-        const examplesStr = nodeData.inputs?.examples as string
+        const examplesStr = nodeData.inputs?.examples
         const prefix = nodeData.inputs?.prefix as string
         const suffix = nodeData.inputs?.suffix as string
         const exampleSeparator = nodeData.inputs?.exampleSeparator as string
@@ -88,7 +88,15 @@ class FewShotPromptTemplate_Prompts implements INode {
         const examplePrompt = nodeData.inputs?.examplePrompt as PromptTemplate
 
         const inputVariables = getInputVariables(suffix)
-        const examples: Example[] = JSON.parse(examplesStr)
+
+        let examples: Example[] = []
+        if (examplesStr) {
+            try {
+                examples = typeof examplesStr === 'object' ? examplesStr : JSON.parse(examplesStr)
+            } catch (exception) {
+                throw new Error("Invalid JSON in the FewShotPromptTemplate's examples: " + exception)
+            }
+        }
 
         try {
             const obj: FewShotPromptTemplateInput = {

packages/components/nodes/prompts/PromptTemplate/PromptTemplate.ts

@@ -43,11 +43,15 @@ class PromptTemplate_Prompts implements INode {
 
     async init(nodeData: INodeData): Promise<any> {
         const template = nodeData.inputs?.template as string
-        const promptValuesStr = nodeData.inputs?.promptValues as string
+        const promptValuesStr = nodeData.inputs?.promptValues
 
         let promptValues: ICommonObject = {}
         if (promptValuesStr) {
-            promptValues = JSON.parse(promptValuesStr)
+            try {
+                promptValues = typeof promptValuesStr === 'object' ? promptValuesStr : JSON.parse(promptValuesStr)
+            } catch (exception) {
+                throw new Error("Invalid JSON in the PromptTemplate's promptValues: " + exception)
+            }
         }
 
         const inputVariables = getInputVariables(template)

packages/components/nodes/textsplitters/CharacterTextSplitter/CharacterTextSplitter.ts

@@ -41,7 +41,7 @@ class CharacterTextSplitter_TextSplitters implements INode {
                 name: 'separator',
                 type: 'string',
                 placeholder: `" "`,
-                description: 'Seperator to determine when to split the text, will override the default separator',
+                description: 'Separator to determine when to split the text, will override the default separator',
                 optional: true
             }
         ]

packages/components/nodes/textsplitters/RecursiveCharacterTextSplitter/RecursiveCharacterTextSplitter.ts

@@ -41,8 +41,9 @@ class RecursiveCharacterTextSplitter_TextSplitters implements INode {
                 name: 'separators',
                 type: 'string',
                 rows: 4,
-                description: 'Array of custom seperators to determine when to split the text, will override the default separators',
+                description: 'Array of custom separators to determine when to split the text, will override the default separators',
                 placeholder: `["|", "##", ">", "-"]`,
+                additionalParams: true,
                 optional: true
             }
         ]
@@ -51,7 +52,7 @@ class RecursiveCharacterTextSplitter_TextSplitters implements INode {
     async init(nodeData: INodeData): Promise<any> {
         const chunkSize = nodeData.inputs?.chunkSize as string
         const chunkOverlap = nodeData.inputs?.chunkOverlap as string
-        const separators = nodeData.inputs?.separators as string
+        const separators = nodeData.inputs?.separators
 
         const obj = {} as RecursiveCharacterTextSplitterParams
 
@@ -59,7 +60,7 @@ class RecursiveCharacterTextSplitter_TextSplitters implements INode {
         if (chunkOverlap) obj.chunkOverlap = parseInt(chunkOverlap, 10)
         if (separators) {
             try {
-                obj.separators = JSON.parse(separators)
+                obj.separators = typeof separators === 'object' ? separators : JSON.parse(separators)
             } catch (e) {
                 throw new Error(e)
             }

packages/components/nodes/tools/SearchApi/SearchAPI.ts

@@ -0,0 +1,42 @@
+import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
+import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
+import { SearchApi } from 'langchain/tools'
+
+class SearchAPI_Tools implements INode {
+    label: string
+    name: string
+    version: number
+    description: string
+    type: string
+    icon: string
+    category: string
+    baseClasses: string[]
+    credential: INodeParams
+    inputs: INodeParams[]
+
+    constructor() {
+        this.label = 'SearchApi'
+        this.name = 'searchAPI'
+        this.version = 1.0
+        this.type = 'SearchAPI'
+        this.icon = 'searchapi.svg'
+        this.category = 'Tools'
+        this.description = 'Real-time API for accessing Google Search data'
+        this.inputs = []
+        this.credential = {
+            label: 'Connect Credential',
+            name: 'credential',
+            type: 'credential',
+            credentialNames: ['searchApi']
+        }
+        this.baseClasses = [this.type, ...getBaseClasses(SearchApi)]
+    }
+
+    async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
+        const credentialData = await getCredentialData(nodeData.credential ?? '', options)
+        const searchApiKey = getCredentialParam('searchApiKey', credentialData, nodeData)
+        return new SearchApi(searchApiKey)
+    }
+}
+
+module.exports = { nodeClass: SearchAPI_Tools }

packages/components/nodes/tools/SearchApi/searchapi.svg

@@ -0,0 +1 @@
+<svg id="SvgjsSvg1001" width="75.27131652832031" height="63.92396926879883" xmlns="http://www.w3.org/2000/svg" version="1.1" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:svgjs="http://svgjs.com/svgjs" viewBox="0 0 75.27131652832031 63.92396926879883"><defs id="SvgjsDefs1002"></defs><rect id="SvgjsRect1008" width="75.27131652832031" height="63.92396926879883" fill="transparent"></rect><g id="SvgjsG1009" transform="matrix(1,0,0,1,-39.50003433227539,-50.53549575805664)"><title>0479_octopus_verti</title><path id="color_1" d="M97.24234,109.8245a2.57759,2.57759,0,0,1-2.57778,2.57778,9.80672,9.80672,0,0,1-9.79558-9.79557V91.58366a2.57779,2.57779,0,0,1,5.15557,0v11.02305a4.64509,4.64509,0,0,0,4.64,4.64A2.57759,2.57759,0,0,1,97.24234,109.8245ZM112.19348,93.223a2.57759,2.57759,0,0,0-2.57778,2.57779,4.64,4.64,0,1,1-9.28,0V73.73554a23.2,23.2,0,1,0-46.40009,0V95.80076a4.64,4.64,0,1,1-9.28,0,2.57779,2.57779,0,1,0-5.15557,0,9.79558,9.79558,0,0,0,19.59115,0V73.73554a18.04449,18.04449,0,0,1,36.089,0V95.80076a9.79558,9.79558,0,0,0,19.59115,0A2.57759,2.57759,0,0,0,112.19348,93.223ZM77.13563,91.78a2.57759,2.57759,0,0,0-2.57778,2.57778v17.52893a2.57779,2.57779,0,0,0,5.15557,0V94.3578A2.57759,2.57759,0,0,0,77.13563,91.78ZM66.8245,89.00588a2.57759,2.57759,0,0,0-2.57778,2.57778v11.02305a4.64509,4.64509,0,0,1-4.64,4.64,2.57778,2.57778,0,1,0,0,5.15556,9.80671,9.80671,0,0,0,9.79557-9.79557V91.58366A2.57759,2.57759,0,0,0,66.8245,89.00588ZM69.918,70.12639a3.6089,3.6089,0,1,0,3.6089,3.6089A3.60891,3.60891,0,0,0,69.918,70.12639Zm18.04448,3.6089a3.6089,3.6089,0,1,0-3.60889,3.60889A3.60891,3.60891,0,0,0,87.96251,73.73529Z" fill="#3730a3"></path></g></svg>

packages/components/nodes/vectorstores/Chroma/Chroma_Upsert.ts

@@ -98,7 +98,9 @@ class ChromaUpsert_VectorStores implements INode {
         const flattenDocs = docs && docs.length ? flatten(docs) : []
         const finalDocs = []
         for (let i = 0; i < flattenDocs.length; i += 1) {
-            finalDocs.push(new Document(flattenDocs[i]))
+            if (flattenDocs[i] && flattenDocs[i].pageContent) {
+                finalDocs.push(new Document(flattenDocs[i]))
+            }
         }
 
         const obj: {

packages/components/nodes/vectorstores/Elasticsearch/Elasticsearch_Upsert.ts

@@ -38,7 +38,9 @@ class ElasicsearchUpsert_VectorStores extends ElasticSearchBase implements INode
         const flattenDocs = docs && docs.length ? flatten(docs) : []
         const finalDocs = []
         for (let i = 0; i < flattenDocs.length; i += 1) {
-            finalDocs.push(new Document(flattenDocs[i]))
+            if (flattenDocs[i] && flattenDocs[i].pageContent) {
+                finalDocs.push(new Document(flattenDocs[i]))
+            }
         }
 
         // The following code is a workaround for a bug (Langchain Issue #1589) in the underlying library.

packages/components/nodes/vectorstores/Faiss_Upsert/Faiss_Upsert.ts

@@ -80,7 +80,9 @@ class FaissUpsert_VectorStores implements INode {
         const flattenDocs = docs && docs.length ? flatten(docs) : []
         const finalDocs = []
         for (let i = 0; i < flattenDocs.length; i += 1) {
-            finalDocs.push(new Document(flattenDocs[i]))
+            if (flattenDocs[i] && flattenDocs[i].pageContent) {
+                finalDocs.push(new Document(flattenDocs[i]))
+            }
         }
 
         const vectorStore = await FaissStore.fromDocuments(finalDocs, embeddings)

packages/components/nodes/vectorstores/InMemory/InMemoryVectorStore.ts

@@ -71,7 +71,9 @@ class InMemoryVectorStore_VectorStores implements INode {
         const flattenDocs = docs && docs.length ? flatten(docs) : []
         const finalDocs = []
         for (let i = 0; i < flattenDocs.length; i += 1) {
-            finalDocs.push(new Document(flattenDocs[i]))
+            if (flattenDocs[i] && flattenDocs[i].pageContent) {
+                finalDocs.push(new Document(flattenDocs[i]))
+            }
         }
 
         const vectorStore = await MemoryVectorStore.fromDocuments(finalDocs, embeddings)

packages/components/nodes/vectorstores/Milvus/Milvus_Upsert.ts

@@ -110,7 +110,9 @@ class Milvus_Upsert_VectorStores implements INode {
         const flattenDocs = docs && docs.length ? flatten(docs) : []
         const finalDocs = []
         for (let i = 0; i < flattenDocs.length; i += 1) {
-            finalDocs.push(new Document(flattenDocs[i]))
+            if (flattenDocs[i] && flattenDocs[i].pageContent) {
+                finalDocs.push(new Document(flattenDocs[i]))
+            }
         }
 
         const vectorStore = await MilvusUpsert.fromDocuments(finalDocs, embeddings, milVusArgs)

packages/components/nodes/vectorstores/OpenSearch_Upsert/OpenSearch_Upsert.ts

@@ -86,7 +86,9 @@ class OpenSearchUpsert_VectorStores implements INode {
         const flattenDocs = docs && docs.length ? flatten(docs) : []
         const finalDocs = []
         for (let i = 0; i < flattenDocs.length; i += 1) {
-            finalDocs.push(new Document(flattenDocs[i]))
+            if (flattenDocs[i] && flattenDocs[i].pageContent) {
+                finalDocs.push(new Document(flattenDocs[i]))
+            }
         }
 
         const client = new Client({

packages/components/nodes/vectorstores/Pinecone/Pinecone_Upsert.ts

@@ -106,7 +106,9 @@ class PineconeUpsert_VectorStores implements INode {
         const flattenDocs = docs && docs.length ? flatten(docs) : []
         const finalDocs = []
         for (let i = 0; i < flattenDocs.length; i += 1) {
-            finalDocs.push(new Document(flattenDocs[i]))
+            if (flattenDocs[i] && flattenDocs[i].pageContent) {
+                finalDocs.push(new Document(flattenDocs[i]))
+            }
         }
 
         const obj: PineconeLibArgs = {

packages/components/nodes/vectorstores/Postgres_Existing/Postgres_Exisiting.ts

@@ -65,6 +65,13 @@ class Postgres_Existing_VectorStores implements INode {
                 additionalParams: true,
                 optional: true
             },
+            {
+                label: 'Additional Configuration',
+                name: 'additionalConfig',
+                type: 'json',
+                additionalParams: true,
+                optional: true
+            },
             {
                 label: 'Top K',
                 name: 'topK',
@@ -96,11 +103,22 @@ class Postgres_Existing_VectorStores implements INode {
         const _tableName = nodeData.inputs?.tableName as string
         const tableName = _tableName ? _tableName : 'documents'
         const embeddings = nodeData.inputs?.embeddings as Embeddings
+        const additionalConfig = nodeData.inputs?.additionalConfig as string
         const output = nodeData.outputs?.output as string
         const topK = nodeData.inputs?.topK as string
         const k = topK ? parseFloat(topK) : 4
 
+        let additionalConfiguration = {}
+        if (additionalConfig) {
+            try {
+                additionalConfiguration = typeof additionalConfig === 'object' ? additionalConfig : JSON.parse(additionalConfig)
+            } catch (exception) {
+                throw new Error('Invalid JSON in the Additional Configuration: ' + exception)
+            }
+        }
+
         const postgresConnectionOptions = {
+            ...additionalConfiguration,
             type: 'postgres',
             host: nodeData.inputs?.host as string,
             port: nodeData.inputs?.port as number,

packages/components/nodes/vectorstores/Postgres_Upsert/Postgres_Upsert.ts

@@ -72,6 +72,13 @@ class PostgresUpsert_VectorStores implements INode {
                 additionalParams: true,
                 optional: true
             },
+            {
+                label: 'Additional Configuration',
+                name: 'additionalConfig',
+                type: 'json',
+                additionalParams: true,
+                optional: true
+            },
             {
                 label: 'Top K',
                 name: 'topK',
@@ -104,11 +111,22 @@ class PostgresUpsert_VectorStores implements INode {
         const tableName = _tableName ? _tableName : 'documents'
         const docs = nodeData.inputs?.document as Document[]
         const embeddings = nodeData.inputs?.embeddings as Embeddings
+        const additionalConfig = nodeData.inputs?.additionalConfig as string
         const output = nodeData.outputs?.output as string
         const topK = nodeData.inputs?.topK as string
         const k = topK ? parseFloat(topK) : 4
 
+        let additionalConfiguration = {}
+        if (additionalConfig) {
+            try {
+                additionalConfiguration = typeof additionalConfig === 'object' ? additionalConfig : JSON.parse(additionalConfig)
+            } catch (exception) {
+                throw new Error('Invalid JSON in the Additional Configuration: ' + exception)
+            }
+        }
+
         const postgresConnectionOptions = {
+            ...additionalConfiguration,
             type: 'postgres',
             host: nodeData.inputs?.host as string,
             port: nodeData.inputs?.port as number,
@@ -125,7 +143,9 @@ class PostgresUpsert_VectorStores implements INode {
         const flattenDocs = docs && docs.length ? flatten(docs) : []
         const finalDocs = []
         for (let i = 0; i < flattenDocs.length; i += 1) {
-            finalDocs.push(new Document(flattenDocs[i]))
+            if (flattenDocs[i] && flattenDocs[i].pageContent) {
+                finalDocs.push(new Document(flattenDocs[i]))
+            }
         }
 
         const vectorStore = await TypeORMVectorStore.fromDocuments(finalDocs, embeddings, args)

packages/components/nodes/vectorstores/Qdrant_Existing/Qdrant_Existing.ts

@@ -23,7 +23,7 @@ class Qdrant_Existing_VectorStores implements INode {
     constructor() {
         this.label = 'Qdrant Load Existing Index'
         this.name = 'qdrantExistingIndex'
-        this.version = 1.0
+        this.version = 2.0
         this.type = 'Qdrant'
         this.icon = 'qdrant.png'
         this.category = 'Vector Stores'
@@ -55,8 +55,39 @@ class Qdrant_Existing_VectorStores implements INode {
                 type: 'string'
             },
             {
-                label: 'Qdrant Collection Cofiguration',
+                label: 'Vector Dimension',
+                name: 'qdrantVectorDimension',
+                type: 'number',
+                default: 1536,
+                additionalParams: true
+            },
+            {
+                label: 'Similarity',
+                name: 'qdrantSimilarity',
+                description: 'Similarity measure used in Qdrant.',
+                type: 'options',
+                default: 'Cosine',
+                options: [
+                    {
+                        label: 'Cosine',
+                        name: 'Cosine'
+                    },
+                    {
+                        label: 'Euclid',
+                        name: 'Euclid'
+                    },
+                    {
+                        label: 'Dot',
+                        name: 'Dot'
+                    }
+                ],
+                additionalParams: true
+            },
+            {
+                label: 'Additional Collection Cofiguration',
                 name: 'qdrantCollectionConfiguration',
+                description:
+                    'Refer to <a target="_blank" href="https://qdrant.tech/documentation/concepts/collections">collection docs</a> for more reference',
                 type: 'json',
                 optional: true,
                 additionalParams: true
@@ -98,6 +129,8 @@ class Qdrant_Existing_VectorStores implements INode {
         const collectionName = nodeData.inputs?.qdrantCollection as string
         let qdrantCollectionConfiguration = nodeData.inputs?.qdrantCollectionConfiguration
         const embeddings = nodeData.inputs?.embeddings as Embeddings
+        const qdrantSimilarity = nodeData.inputs?.qdrantSimilarity
+        const qdrantVectorDimension = nodeData.inputs?.qdrantVectorDimension
         const output = nodeData.outputs?.output as string
         const topK = nodeData.inputs?.topK as string
         let queryFilter = nodeData.inputs?.queryFilter
@@ -126,7 +159,14 @@ class Qdrant_Existing_VectorStores implements INode {
                 typeof qdrantCollectionConfiguration === 'object'
                     ? qdrantCollectionConfiguration
                     : JSON.parse(qdrantCollectionConfiguration)
-            dbConfig.collectionConfig = qdrantCollectionConfiguration
+            dbConfig.collectionConfig = {
+                ...qdrantCollectionConfiguration,
+                vectors: {
+                    ...qdrantCollectionConfiguration.vectors,
+                    size: qdrantVectorDimension ? parseInt(qdrantVectorDimension, 10) : 1536,
+                    distance: qdrantSimilarity ?? 'Cosine'
+                }
+            }
         }
 
         if (queryFilter) {

packages/components/nodes/vectorstores/Qdrant_Upsert/Qdrant_Upsert.ts

@@ -25,7 +25,7 @@ class QdrantUpsert_VectorStores implements INode {
     constructor() {
         this.label = 'Qdrant Upsert Document'
         this.name = 'qdrantUpsert'
-        this.version = 1.0
+        this.version = 2.0
         this.type = 'Qdrant'
         this.icon = 'qdrant.png'
         this.category = 'Vector Stores'
@@ -62,6 +62,35 @@ class QdrantUpsert_VectorStores implements INode {
                 name: 'qdrantCollection',
                 type: 'string'
             },
+            {
+                label: 'Vector Dimension',
+                name: 'qdrantVectorDimension',
+                type: 'number',
+                default: 1536,
+                additionalParams: true
+            },
+            {
+                label: 'Similarity',
+                name: 'qdrantSimilarity',
+                description: 'Similarity measure used in Qdrant.',
+                type: 'options',
+                default: 'Cosine',
+                options: [
+                    {
+                        label: 'Cosine',
+                        name: 'Cosine'
+                    },
+                    {
+                        label: 'Euclid',
+                        name: 'Euclid'
+                    },
+                    {
+                        label: 'Dot',
+                        name: 'Dot'
+                    }
+                ],
+                additionalParams: true
+            },
             {
                 label: 'Top K',
                 name: 'topK',
@@ -99,6 +128,9 @@ class QdrantUpsert_VectorStores implements INode {
         const collectionName = nodeData.inputs?.qdrantCollection as string
         const docs = nodeData.inputs?.document as Document[]
         const embeddings = nodeData.inputs?.embeddings as Embeddings
+        const qdrantSimilarity = nodeData.inputs?.qdrantSimilarity
+        const qdrantVectorDimension = nodeData.inputs?.qdrantVectorDimension
+
         const output = nodeData.outputs?.output as string
         const topK = nodeData.inputs?.topK as string
         const k = topK ? parseFloat(topK) : 4
@@ -115,13 +147,21 @@ class QdrantUpsert_VectorStores implements INode {
         const flattenDocs = docs && docs.length ? flatten(docs) : []
         const finalDocs = []
         for (let i = 0; i < flattenDocs.length; i += 1) {
-            finalDocs.push(new Document(flattenDocs[i]))
+            if (flattenDocs[i] && flattenDocs[i].pageContent) {
+                finalDocs.push(new Document(flattenDocs[i]))
+            }
         }
 
         const dbConfig: QdrantLibArgs = {
             client,
             url: qdrantServerUrl,
-            collectionName
+            collectionName,
+            collectionConfig: {
+                vectors: {
+                    size: qdrantVectorDimension ? parseInt(qdrantVectorDimension, 10) : 1536,
+                    distance: qdrantSimilarity ?? 'Cosine'
+                }
+            }
         }
 
         const retrieverConfig: RetrieverConfig = {

packages/components/nodes/vectorstores/Redis/RedisSearchBase.ts

@@ -0,0 +1,215 @@
+import {
+    getBaseClasses,
+    getCredentialData,
+    getCredentialParam,
+    ICommonObject,
+    INodeData,
+    INodeOutputsValue,
+    INodeParams
+} from '../../../src'
+
+import { Embeddings } from 'langchain/embeddings/base'
+import { VectorStore } from 'langchain/vectorstores/base'
+import { Document } from 'langchain/document'
+import { createClient, SearchOptions } from 'redis'
+import { RedisVectorStore } from 'langchain/vectorstores/redis'
+import { escapeSpecialChars, unEscapeSpecialChars } from './utils'
+
+export abstract class RedisSearchBase {
+    label: string
+    name: string
+    version: number
+    description: string
+    type: string
+    icon: string
+    category: string
+    baseClasses: string[]
+    inputs: INodeParams[]
+    credential: INodeParams
+    outputs: INodeOutputsValue[]
+    redisClient: ReturnType<typeof createClient>
+
+    protected constructor() {
+        this.type = 'Redis'
+        this.icon = 'redis.svg'
+        this.category = 'Vector Stores'
+        this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
+        this.credential = {
+            label: 'Connect Credential',
+            name: 'credential',
+            type: 'credential',
+            credentialNames: ['redisCacheUrlApi', 'redisCacheApi']
+        }
+        this.inputs = [
+            {
+                label: 'Embeddings',
+                name: 'embeddings',
+                type: 'Embeddings'
+            },
+            {
+                label: 'Index Name',
+                name: 'indexName',
+                placeholder: '<VECTOR_INDEX_NAME>',
+                type: 'string'
+            },
+            {
+                label: 'Replace Index?',
+                name: 'replaceIndex',
+                description: 'Selecting this option will delete the existing index and recreate a new one',
+                default: false,
+                type: 'boolean'
+            },
+            {
+                label: 'Content Field',
+                name: 'contentKey',
+                description: 'Name of the field (column) that contains the actual content',
+                type: 'string',
+                default: 'content',
+                additionalParams: true,
+                optional: true
+            },
+            {
+                label: 'Metadata Field',
+                name: 'metadataKey',
+                description: 'Name of the field (column) that contains the metadata of the document',
+                type: 'string',
+                default: 'metadata',
+                additionalParams: true,
+                optional: true
+            },
+            {
+                label: 'Vector Field',
+                name: 'vectorKey',
+                description: 'Name of the field (column) that contains the vector',
+                type: 'string',
+                default: 'content_vector',
+                additionalParams: true,
+                optional: true
+            },
+            {
+                label: 'Top K',
+                name: 'topK',
+                description: 'Number of top results to fetch. Default to 4',
+                placeholder: '4',
+                type: 'number',
+                additionalParams: true,
+                optional: true
+            }
+        ]
+        this.outputs = [
+            {
+                label: 'Redis Retriever',
+                name: 'retriever',
+                baseClasses: this.baseClasses
+            },
+            {
+                label: 'Redis Vector Store',
+                name: 'vectorStore',
+                baseClasses: [this.type, ...getBaseClasses(RedisVectorStore)]
+            }
+        ]
+    }
+
+    abstract constructVectorStore(
+        embeddings: Embeddings,
+        indexName: string,
+        replaceIndex: boolean,
+        docs: Document<Record<string, any>>[] | undefined
+    ): Promise<VectorStore>
+
+    async init(nodeData: INodeData, _: string, options: ICommonObject, docs: Document<Record<string, any>>[] | undefined): Promise<any> {
+        const credentialData = await getCredentialData(nodeData.credential ?? '', options)
+        const indexName = nodeData.inputs?.indexName as string
+        let contentKey = nodeData.inputs?.contentKey as string
+        let metadataKey = nodeData.inputs?.metadataKey as string
+        let vectorKey = nodeData.inputs?.vectorKey as string
+        const embeddings = nodeData.inputs?.embeddings as Embeddings
+        const topK = nodeData.inputs?.topK as string
+        const replaceIndex = nodeData.inputs?.replaceIndex as boolean
+        const k = topK ? parseFloat(topK) : 4
+        const output = nodeData.outputs?.output as string
+
+        let redisUrl = getCredentialParam('redisUrl', credentialData, nodeData)
+        if (!redisUrl || redisUrl === '') {
+            const username = getCredentialParam('redisCacheUser', credentialData, nodeData)
+            const password = getCredentialParam('redisCachePwd', credentialData, nodeData)
+            const portStr = getCredentialParam('redisCachePort', credentialData, nodeData)
+            const host = getCredentialParam('redisCacheHost', credentialData, nodeData)
+
+            redisUrl = 'redis://' + username + ':' + password + '@' + host + ':' + portStr
+        }
+
+        this.redisClient = createClient({ url: redisUrl })
+        await this.redisClient.connect()
+
+        const vectorStore = await this.constructVectorStore(embeddings, indexName, replaceIndex, docs)
+        if (!contentKey || contentKey === '') contentKey = 'content'
+        if (!metadataKey || metadataKey === '') metadataKey = 'metadata'
+        if (!vectorKey || vectorKey === '') vectorKey = 'content_vector'
+
+        const buildQuery = (query: number[], k: number, filter?: string[]): [string, SearchOptions] => {
+            const vectorScoreField = 'vector_score'
+
+            let hybridFields = '*'
+            // if a filter is set, modify the hybrid query
+            if (filter && filter.length) {
+                // `filter` is a list of strings, then it's applied using the OR operator in the metadata key
+                hybridFields = `@${metadataKey}:(${filter.map(escapeSpecialChars).join('|')})`
+            }
+
+            const baseQuery = `${hybridFields} => [KNN ${k} @${vectorKey} $vector AS ${vectorScoreField}]`
+            const returnFields = [metadataKey, contentKey, vectorScoreField]
+
+            const options: SearchOptions = {
+                PARAMS: {
+                    vector: Buffer.from(new Float32Array(query).buffer)
+                },
+                RETURN: returnFields,
+                SORTBY: vectorScoreField,
+                DIALECT: 2,
+                LIMIT: {
+                    from: 0,
+                    size: k
+                }
+            }
+
+            return [baseQuery, options]
+        }
+
+        vectorStore.similaritySearchVectorWithScore = async (
+            query: number[],
+            k: number,
+            filter?: string[]
+        ): Promise<[Document, number][]> => {
+            const results = await this.redisClient.ft.search(indexName, ...buildQuery(query, k, filter))
+            const result: [Document, number][] = []
+
+            if (results.total) {
+                for (const res of results.documents) {
+                    if (res.value) {
+                        const document = res.value
+                        if (document.vector_score) {
+                            const metadataString = unEscapeSpecialChars(document[metadataKey] as string)
+                            result.push([
+                                new Document({
+                                    pageContent: document[contentKey] as string,
+                                    metadata: JSON.parse(metadataString)
+                                }),
+                                Number(document.vector_score)
+                            ])
+                        }
+                    }
+                }
+            }
+            return result
+        }
+
+        if (output === 'retriever') {
+            return vectorStore.asRetriever(k)
+        } else if (output === 'vectorStore') {
+            ;(vectorStore as any).k = k
+            return vectorStore
+        }
+        return vectorStore
+    }
+}

packages/components/nodes/vectorstores/Redis/Redis_Existing.ts

@@ -0,0 +1,42 @@
+import { ICommonObject, INode, INodeData } from '../../../src/Interface'
+import { Embeddings } from 'langchain/embeddings/base'
+import { VectorStore } from 'langchain/vectorstores/base'
+import { RedisVectorStore, RedisVectorStoreConfig } from 'langchain/vectorstores/redis'
+import { Document } from 'langchain/document'
+
+import { RedisSearchBase } from './RedisSearchBase'
+
+class RedisExisting_VectorStores extends RedisSearchBase implements INode {
+    constructor() {
+        super()
+        this.label = 'Redis Load Existing Index'
+        this.name = 'RedisIndex'
+        this.version = 1.0
+        this.description = 'Load existing index from Redis (i.e: Document has been upserted)'
+
+        // Remove deleteIndex from inputs as it is not applicable while fetching data from Redis
+        let input = this.inputs.find((i) => i.name === 'deleteIndex')
+        if (input) this.inputs.splice(this.inputs.indexOf(input), 1)
+    }
+
+    async constructVectorStore(
+        embeddings: Embeddings,
+        indexName: string,
+        // eslint-disable-next-line unused-imports/no-unused-vars
+        replaceIndex: boolean,
+        _: Document<Record<string, any>>[]
+    ): Promise<VectorStore> {
+        const storeConfig: RedisVectorStoreConfig = {
+            redisClient: this.redisClient,
+            indexName: indexName
+        }
+
+        return new RedisVectorStore(embeddings, storeConfig)
+    }
+
+    async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
+        return super.init(nodeData, _, options, undefined)
+    }
+}
+
+module.exports = { nodeClass: RedisExisting_VectorStores }

packages/components/nodes/vectorstores/Redis/Redis_Upsert.ts

@@ -0,0 +1,63 @@
+import { ICommonObject, INode, INodeData } from '../../../src/Interface'
+import { Embeddings } from 'langchain/embeddings/base'
+import { Document } from 'langchain/document'
+
+import { flatten } from 'lodash'
+import { RedisSearchBase } from './RedisSearchBase'
+import { VectorStore } from 'langchain/vectorstores/base'
+import { RedisVectorStore, RedisVectorStoreConfig } from 'langchain/vectorstores/redis'
+import { escapeAllStrings } from './utils'
+
+class RedisUpsert_VectorStores extends RedisSearchBase implements INode {
+    constructor() {
+        super()
+        this.label = 'Redis Upsert Document'
+        this.name = 'RedisUpsert'
+        this.version = 1.0
+        this.description = 'Upsert documents to Redis'
+        this.inputs.unshift({
+            label: 'Document',
+            name: 'document',
+            type: 'Document',
+            list: true
+        })
+    }
+
+    async constructVectorStore(
+        embeddings: Embeddings,
+        indexName: string,
+        replaceIndex: boolean,
+        docs: Document<Record<string, any>>[]
+    ): Promise<VectorStore> {
+        const storeConfig: RedisVectorStoreConfig = {
+            redisClient: this.redisClient,
+            indexName: indexName
+        }
+        if (replaceIndex) {
+            let response = await this.redisClient.ft.dropIndex(indexName)
+            if (process.env.DEBUG === 'true') {
+                // eslint-disable-next-line no-console
+                console.log(`Redis Vector Store :: Dropping index [${indexName}], Received Response [${response}]`)
+            }
+        }
+        return await RedisVectorStore.fromDocuments(docs, embeddings, storeConfig)
+    }
+
+    async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
+        const docs = nodeData.inputs?.document as Document[]
+
+        const flattenDocs = docs && docs.length ? flatten(docs) : []
+        const finalDocs = []
+        for (let i = 0; i < flattenDocs.length; i += 1) {
+            if (flattenDocs[i] && flattenDocs[i].pageContent) {
+                const document = new Document(flattenDocs[i])
+                escapeAllStrings(document.metadata)
+                finalDocs.push(document)
+            }
+        }
+
+        return super.init(nodeData, _, options, flattenDocs)
+    }
+}
+
+module.exports = { nodeClass: RedisUpsert_VectorStores }

packages/components/nodes/vectorstores/Redis/redis.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 128 128" id="redis"><path fill="#A41E11" d="M121.8 93.1c-6.7 3.5-41.4 17.7-48.8 21.6-7.4 3.9-11.5 3.8-17.3 1s-42.7-17.6-49.4-20.8c-3.3-1.6-5-2.9-5-4.2v-12.7s48-10.5 55.8-13.2c7.8-2.8 10.4-2.9 17-.5s46.1 9.5 52.6 11.9v12.5c0 1.3-1.5 2.7-4.9 4.4z"></path><path fill="#D82C20" d="M121.8 80.5c-6.7 3.5-41.4 17.7-48.8 21.6-7.4 3.9-11.5 3.8-17.3 1-5.8-2.8-42.7-17.7-49.4-20.9-6.6-3.2-6.8-5.4-.3-7.9 6.5-2.6 43.2-17 51-19.7 7.8-2.8 10.4-2.9 17-.5s41.1 16.1 47.6 18.5c6.7 2.4 6.9 4.4.2 7.9z"></path><path fill="#A41E11" d="M121.8 72.5c-6.7 3.5-41.4 17.7-48.8 21.6-7.4 3.8-11.5 3.8-17.3 1-5.8-2.8-42.7-17.7-49.4-20.9-3.3-1.6-5-2.9-5-4.2v-12.7s48-10.5 55.8-13.2c7.8-2.8 10.4-2.9 17-.5s46.1 9.5 52.6 11.9v12.5c0 1.3-1.5 2.7-4.9 4.5z"></path><path fill="#D82C20" d="M121.8 59.8c-6.7 3.5-41.4 17.7-48.8 21.6-7.4 3.8-11.5 3.8-17.3 1-5.8-2.8-42.7-17.7-49.4-20.9s-6.8-5.4-.3-7.9c6.5-2.6 43.2-17 51-19.7 7.8-2.8 10.4-2.9 17-.5s41.1 16.1 47.6 18.5c6.7 2.4 6.9 4.4.2 7.9z"></path><path fill="#A41E11" d="M121.8 51c-6.7 3.5-41.4 17.7-48.8 21.6-7.4 3.8-11.5 3.8-17.3 1-5.8-2.7-42.7-17.6-49.4-20.8-3.3-1.6-5.1-2.9-5.1-4.2v-12.7s48-10.5 55.8-13.2c7.8-2.8 10.4-2.9 17-.5s46.1 9.5 52.6 11.9v12.5c.1 1.3-1.4 2.6-4.8 4.4z"></path><path fill="#D82C20" d="M121.8 38.3c-6.7 3.5-41.4 17.7-48.8 21.6-7.4 3.8-11.5 3.8-17.3 1s-42.7-17.6-49.4-20.8-6.8-5.4-.3-7.9c6.5-2.6 43.2-17 51-19.7 7.8-2.8 10.4-2.9 17-.5s41.1 16.1 47.6 18.5c6.7 2.4 6.9 4.4.2 7.8z"></path><path fill="#fff" d="M80.4 26.1l-10.8 1.2-2.5 5.8-3.9-6.5-12.5-1.1 9.3-3.4-2.8-5.2 8.8 3.4 8.2-2.7-2.2 5.4zM66.5 54.5l-20.3-8.4 29.1-4.4z"></path><ellipse cx="38.4" cy="35.4" fill="#fff" rx="15.5" ry="6"></ellipse><path fill="#7A0C00" d="M93.3 27.7l17.2 6.8-17.2 6.8z"></path><path fill="#AD2115" d="M74.3 35.3l19-7.6v13.6l-1.9.8z"></path></svg>

packages/components/nodes/vectorstores/Redis/utils.ts

@@ -0,0 +1,25 @@
+/*
+ * Escapes all '-' characters.
+ * Redis Search considers '-' as a negative operator, hence we need
+ * to escape it
+ */
+export const escapeSpecialChars = (str: string) => {
+    return str.replaceAll('-', '\\-')
+}
+
+export const escapeAllStrings = (obj: object) => {
+    Object.keys(obj).forEach((key: string) => {
+        // @ts-ignore
+        let item = obj[key]
+        if (typeof item === 'object') {
+            escapeAllStrings(item)
+        } else if (typeof item === 'string') {
+            // @ts-ignore
+            obj[key] = escapeSpecialChars(item)
+        }
+    })
+}
+
+export const unEscapeSpecialChars = (str: string) => {
+    return str.replaceAll('\\-', '-')
+}

packages/components/nodes/vectorstores/Singlestore_Upsert/Singlestore_Upsert.ts

@@ -140,7 +140,9 @@ class SingleStoreUpsert_VectorStores implements INode {
         const flattenDocs = docs && docs.length ? flatten(docs) : []
         const finalDocs = []
         for (let i = 0; i < flattenDocs.length; i += 1) {
-            finalDocs.push(new Document(flattenDocs[i]))
+            if (flattenDocs[i] && flattenDocs[i].pageContent) {
+                finalDocs.push(new Document(flattenDocs[i]))
+            }
         }
 
         let vectorStore: SingleStoreVectorStore

packages/components/nodes/vectorstores/Vectara/Vectara_Upsert.ts

@@ -132,7 +132,9 @@ class VectaraUpsert_VectorStores implements INode {
         const flattenDocs = docs && docs.length ? flatten(docs) : []
         const finalDocs = []
         for (let i = 0; i < flattenDocs.length; i += 1) {
-            finalDocs.push(new Document(flattenDocs[i]))
+            if (flattenDocs[i] && flattenDocs[i].pageContent) {
+                finalDocs.push(new Document(flattenDocs[i]))
+            }
         }
 
         const vectorStore = await VectaraStore.fromDocuments(finalDocs, embeddings, vectaraArgs)

packages/components/nodes/vectorstores/Weaviate_Upsert/Weaviate_Upsert.ts

@@ -143,7 +143,9 @@ class WeaviateUpsert_VectorStores implements INode {
         const flattenDocs = docs && docs.length ? flatten(docs) : []
         const finalDocs = []
         for (let i = 0; i < flattenDocs.length; i += 1) {
-            finalDocs.push(new Document(flattenDocs[i]))
+            if (flattenDocs[i] && flattenDocs[i].pageContent) {
+                finalDocs.push(new Document(flattenDocs[i]))
+            }
         }
 
         const obj: WeaviateLibArgs = {

packages/components/nodes/vectorstores/Zep/Zep_Upsert.ts

@@ -106,7 +106,9 @@ class Zep_Upsert_VectorStores implements INode {
         const flattenDocs = docs && docs.length ? flatten(docs) : []
         const finalDocs = []
         for (let i = 0; i < flattenDocs.length; i += 1) {
-            finalDocs.push(new Document(flattenDocs[i]))
+            if (flattenDocs[i] && flattenDocs[i].pageContent) {
+                finalDocs.push(new Document(flattenDocs[i]))
+            }
         }
 
         const zepConfig: IZepConfig = {

packages/components/src/utils.ts

@@ -133,6 +133,7 @@ export const getNodeModulesPackagePath = (packageName: string): string => {
  * @returns {boolean}
  */
 export const getInputVariables = (paramValue: string): string[] => {
+    if (typeof paramValue !== 'string') return []
     let returnVal = paramValue
     const variableStack = []
     const inputVariables = []
@@ -302,7 +303,7 @@ async function crawl(baseURL: string, currentURL: string, pages: string[], limit
 }
 
 /**
- * Prep URL before passing into recursive carwl function
+ * Prep URL before passing into recursive crawl function
  * @param {string} stringURL
  * @param {number} limit
  * @returns {Promise<string[]>}
@@ -446,7 +447,7 @@ export const getCredentialData = async (selectedCredentialId: string, options: I
 
         if (!credential) return {}
 
-        // Decrpyt credentialData
+        // Decrypt credentialData
         const decryptedCredentialData = await decryptCredentialData(credential.encryptedData)
 
         return decryptedCredentialData

packages/server/marketplaces/chatflows/Antonym.json

@@ -45,7 +45,7 @@
                         "id": "fewShotPromptTemplate_1-input-suffix-string"
                     },
                     {
-                        "label": "Example Seperator",
+                        "label": "Example Separator",
                         "name": "exampleSeparator",
                         "type": "string",
                         "placeholder": "\n\n",

packages/server/marketplaces/chatflows/Chat with a Podcast.json

@@ -0,0 +1,669 @@
+{
+    "description": "Engage with data sources such as YouTube Transcripts, Google, and more through intelligent Q&A interactions",
+    "nodes": [
+        {
+            "width": 300,
+            "height": 483,
+            "id": "conversationalRetrievalQAChain_0",
+            "position": {
+                "x": 1499.2693059023254,
+                "y": 430.03911199833317
+            },
+            "type": "customNode",
+            "data": {
+                "id": "conversationalRetrievalQAChain_0",
+                "label": "Conversational Retrieval QA Chain",
+                "version": 1,
+                "name": "conversationalRetrievalQAChain",
+                "type": "ConversationalRetrievalQAChain",
+                "baseClasses": ["ConversationalRetrievalQAChain", "BaseChain", "Runnable"],
+                "category": "Chains",
+                "description": "Document QA - built on RetrievalQAChain to provide a chat history component",
+                "inputParams": [
+                    {
+                        "label": "Return Source Documents",
+                        "name": "returnSourceDocuments",
+                        "type": "boolean",
+                        "optional": true,
+                        "id": "conversationalRetrievalQAChain_0-input-returnSourceDocuments-boolean"
+                    },
+                    {
+                        "label": "System Message",
+                        "name": "systemMessagePrompt",
+                        "type": "string",
+                        "rows": 4,
+                        "additionalParams": true,
+                        "optional": true,
+                        "placeholder": "I want you to act as a document that I am having a conversation with. Your name is \"AI Assistant\". You will provide me with answers from the given info. If the answer is not included, say exactly \"Hmm, I am not sure.\" and stop after that. Refuse to answer any question not about the info. Never break character.",
+                        "id": "conversationalRetrievalQAChain_0-input-systemMessagePrompt-string"
+                    },
+                    {
+                        "label": "Chain Option",
+                        "name": "chainOption",
+                        "type": "options",
+                        "options": [
+                            {
+                                "label": "MapReduceDocumentsChain",
+                                "name": "map_reduce",
+                                "description": "Suitable for QA tasks over larger documents and can run the preprocessing step in parallel, reducing the running time"
+                            },
+                            {
+                                "label": "RefineDocumentsChain",
+                                "name": "refine",
+                                "description": "Suitable for QA tasks over a large number of documents."
+                            },
+                            {
+                                "label": "StuffDocumentsChain",
+                                "name": "stuff",
+                                "description": "Suitable for QA tasks over a small number of documents."
+                            }
+                        ],
+                        "additionalParams": true,
+                        "optional": true,
+                        "id": "conversationalRetrievalQAChain_0-input-chainOption-options"
+                    }
+                ],
+                "inputAnchors": [
+                    {
+                        "label": "Language Model",
+                        "name": "model",
+                        "type": "BaseLanguageModel",
+                        "id": "conversationalRetrievalQAChain_0-input-model-BaseLanguageModel"
+                    },
+                    {
+                        "label": "Vector Store Retriever",
+                        "name": "vectorStoreRetriever",
+                        "type": "BaseRetriever",
+                        "id": "conversationalRetrievalQAChain_0-input-vectorStoreRetriever-BaseRetriever"
+                    },
+                    {
+                        "label": "Memory",
+                        "name": "memory",
+                        "type": "BaseMemory",
+                        "optional": true,
+                        "description": "If left empty, a default BufferMemory will be used",
+                        "id": "conversationalRetrievalQAChain_0-input-memory-BaseMemory"
+                    }
+                ],
+                "inputs": {
+                    "model": "{{chatOpenAI_0.data.instance}}",
+                    "vectorStoreRetriever": "{{memoryVectorStore_0.data.instance}}",
+                    "memory": "",
+                    "returnSourceDocuments": "",
+                    "systemMessagePrompt": "",
+                    "chainOption": ""
+                },
+                "outputAnchors": [
+                    {
+                        "id": "conversationalRetrievalQAChain_0-output-conversationalRetrievalQAChain-ConversationalRetrievalQAChain|BaseChain|Runnable",
+                        "name": "conversationalRetrievalQAChain",
+                        "label": "ConversationalRetrievalQAChain",
+                        "type": "ConversationalRetrievalQAChain | BaseChain | Runnable"
+                    }
+                ],
+                "outputs": {},
+                "selected": false
+            },
+            "selected": false,
+            "positionAbsolute": {
+                "x": 1499.2693059023254,
+                "y": 430.03911199833317
+            },
+            "dragging": false
+        },
+        {
+            "width": 300,
+            "height": 408,
+            "id": "memoryVectorStore_0",
+            "position": {
+                "x": 1082.0280622332507,
+                "y": 589.9990964387842
+            },
+            "type": "customNode",
+            "data": {
+                "id": "memoryVectorStore_0",
+                "label": "In-Memory Vector Store",
+                "version": 1,
+                "name": "memoryVectorStore",
+                "type": "Memory",
+                "baseClasses": ["Memory", "VectorStoreRetriever", "BaseRetriever"],
+                "category": "Vector Stores",
+                "description": "In-memory vectorstore that stores embeddings and does an exact, linear search for the most similar embeddings.",
+                "inputParams": [
+                    {
+                        "label": "Top K",
+                        "name": "topK",
+                        "description": "Number of top results to fetch. Default to 4",
+                        "placeholder": "4",
+                        "type": "number",
+                        "optional": true,
+                        "id": "memoryVectorStore_0-input-topK-number"
+                    }
+                ],
+                "inputAnchors": [
+                    {
+                        "label": "Document",
+                        "name": "document",
+                        "type": "Document",
+                        "list": true,
+                        "id": "memoryVectorStore_0-input-document-Document"
+                    },
+                    {
+                        "label": "Embeddings",
+                        "name": "embeddings",
+                        "type": "Embeddings",
+                        "id": "memoryVectorStore_0-input-embeddings-Embeddings"
+                    }
+                ],
+                "inputs": {
+                    "document": ["{{searchApi_0.data.instance}}", "{{searchApi_0.data.instance}}", "{{searchApi_0.data.instance}}"],
+                    "embeddings": "{{openAIEmbeddings_0.data.instance}}",
+                    "topK": ""
+                },
+                "outputAnchors": [
+                    {
+                        "name": "output",
+                        "label": "Output",
+                        "type": "options",
+                        "options": [
+                            {
+                                "id": "memoryVectorStore_0-output-retriever-Memory|VectorStoreRetriever|BaseRetriever",
+                                "name": "retriever",
+                                "label": "Memory Retriever",
+                                "type": "Memory | VectorStoreRetriever | BaseRetriever"
+                            },
+                            {
+                                "id": "memoryVectorStore_0-output-vectorStore-Memory|VectorStore",
+                                "name": "vectorStore",
+                                "label": "Memory Vector Store",
+                                "type": "Memory | VectorStore"
+                            }
+                        ],
+                        "default": "retriever"
+                    }
+                ],
+                "outputs": {
+                    "output": "retriever"
+                },
+                "selected": false
+            },
+            "positionAbsolute": {
+                "x": 1082.0280622332507,
+                "y": 589.9990964387842
+            },
+            "selected": false,
+            "dragging": false
+        },
+        {
+            "width": 300,
+            "height": 577,
+            "id": "chatOpenAI_0",
+            "position": {
+                "x": 1056.2788608917747,
+                "y": -60.59149112477064
+            },
+            "type": "customNode",
+            "data": {
+                "id": "chatOpenAI_0",
+                "label": "ChatOpenAI",
+                "version": 2,
+                "name": "chatOpenAI",
+                "type": "ChatOpenAI",
+                "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel", "Runnable"],
+                "category": "Chat Models",
+                "description": "Wrapper around OpenAI large language models that use the Chat endpoint",
+                "inputParams": [
+                    {
+                        "label": "Connect Credential",
+                        "name": "credential",
+                        "type": "credential",
+                        "credentialNames": ["openAIApi"],
+                        "id": "chatOpenAI_0-input-credential-credential"
+                    },
+                    {
+                        "label": "Model Name",
+                        "name": "modelName",
+                        "type": "options",
+                        "options": [
+                            {
+                                "label": "gpt-4",
+                                "name": "gpt-4"
+                            },
+                            {
+                                "label": "gpt-4-0613",
+                                "name": "gpt-4-0613"
+                            },
+                            {
+                                "label": "gpt-4-32k",
+                                "name": "gpt-4-32k"
+                            },
+                            {
+                                "label": "gpt-4-32k-0613",
+                                "name": "gpt-4-32k-0613"
+                            },
+                            {
+                                "label": "gpt-3.5-turbo",
+                                "name": "gpt-3.5-turbo"
+                            },
+                            {
+                                "label": "gpt-3.5-turbo-0613",
+                                "name": "gpt-3.5-turbo-0613"
+                            },
+                            {
+                                "label": "gpt-3.5-turbo-16k",
+                                "name": "gpt-3.5-turbo-16k"
+                            },
+                            {
+                                "label": "gpt-3.5-turbo-16k-0613",
+                                "name": "gpt-3.5-turbo-16k-0613"
+                            }
+                        ],
+                        "default": "gpt-3.5-turbo",
+                        "optional": true,
+                        "id": "chatOpenAI_0-input-modelName-options"
+                    },
+                    {
+                        "label": "Temperature",
+                        "name": "temperature",
+                        "type": "number",
+                        "step": 0.1,
+                        "default": 0.9,
+                        "optional": true,
+                        "id": "chatOpenAI_0-input-temperature-number"
+                    },
+                    {
+                        "label": "Max Tokens",
+                        "name": "maxTokens",
+                        "type": "number",
+                        "step": 1,
+                        "optional": true,
+                        "additionalParams": true,
+                        "id": "chatOpenAI_0-input-maxTokens-number"
+                    },
+                    {
+                        "label": "Top Probability",
+                        "name": "topP",
+                        "type": "number",
+                        "step": 0.1,
+                        "optional": true,
+                        "additionalParams": true,
+                        "id": "chatOpenAI_0-input-topP-number"
+                    },
+                    {
+                        "label": "Frequency Penalty",
+                        "name": "frequencyPenalty",
+                        "type": "number",
+                        "step": 0.1,
+                        "optional": true,
+                        "additionalParams": true,
+                        "id": "chatOpenAI_0-input-frequencyPenalty-number"
+                    },
+                    {
+                        "label": "Presence Penalty",
+                        "name": "presencePenalty",
+                        "type": "number",
+                        "step": 0.1,
+                        "optional": true,
+                        "additionalParams": true,
+                        "id": "chatOpenAI_0-input-presencePenalty-number"
+                    },
+                    {
+                        "label": "Timeout",
+                        "name": "timeout",
+                        "type": "number",
+                        "step": 1,
+                        "optional": true,
+                        "additionalParams": true,
+                        "id": "chatOpenAI_0-input-timeout-number"
+                    },
+                    {
+                        "label": "BasePath",
+                        "name": "basepath",
+                        "type": "string",
+                        "optional": true,
+                        "additionalParams": true,
+                        "id": "chatOpenAI_0-input-basepath-string"
+                    },
+                    {
+                        "label": "BaseOptions",
+                        "name": "baseOptions",
+                        "type": "json",
+                        "optional": true,
+                        "additionalParams": true,
+                        "id": "chatOpenAI_0-input-baseOptions-json"
+                    }
+                ],
+                "inputAnchors": [
+                    {
+                        "label": "Cache",
+                        "name": "cache",
+                        "type": "BaseCache",
+                        "optional": true,
+                        "id": "chatOpenAI_0-input-cache-BaseCache"
+                    }
+                ],
+                "inputs": {
+                    "cache": "",
+                    "modelName": "gpt-3.5-turbo",
+                    "temperature": "0.5",
+                    "maxTokens": "",
+                    "topP": "",
+                    "frequencyPenalty": "",
+                    "presencePenalty": "",
+                    "timeout": "",
+                    "basepath": "",
+                    "baseOptions": ""
+                },
+                "outputAnchors": [
+                    {
+                        "id": "chatOpenAI_0-output-chatOpenAI-ChatOpenAI|BaseChatModel|BaseLanguageModel|Runnable",
+                        "name": "chatOpenAI",
+                        "label": "ChatOpenAI",
+                        "type": "ChatOpenAI | BaseChatModel | BaseLanguageModel | Runnable"
+                    }
+                ],
+                "outputs": {},
+                "selected": false
+            },
+            "selected": false,
+            "positionAbsolute": {
+                "x": 1056.2788608917747,
+                "y": -60.59149112477064
+            },
+            "dragging": false
+        },
+        {
+            "width": 300,
+            "height": 478,
+            "id": "characterTextSplitter_0",
+            "position": {
+                "x": 260.5475803279806,
+                "y": -65.1647664861618
+            },
+            "type": "customNode",
+            "data": {
+                "id": "characterTextSplitter_0",
+                "label": "Character Text Splitter",
+                "version": 1,
+                "name": "characterTextSplitter",
+                "type": "CharacterTextSplitter",
+                "baseClasses": ["CharacterTextSplitter", "TextSplitter", "BaseDocumentTransformer", "Runnable"],
+                "category": "Text Splitters",
+                "description": "splits only on one type of character (defaults to \"\\n\\n\").",
+                "inputParams": [
+                    {
+                        "label": "Chunk Size",
+                        "name": "chunkSize",
+                        "type": "number",
+                        "default": 1000,
+                        "optional": true,
+                        "id": "characterTextSplitter_0-input-chunkSize-number"
+                    },
+                    {
+                        "label": "Chunk Overlap",
+                        "name": "chunkOverlap",
+                        "type": "number",
+                        "optional": true,
+                        "id": "characterTextSplitter_0-input-chunkOverlap-number"
+                    },
+                    {
+                        "label": "Custom Separator",
+                        "name": "separator",
+                        "type": "string",
+                        "placeholder": "\" \"",
+                        "description": "Seperator to determine when to split the text, will override the default separator",
+                        "optional": true,
+                        "id": "characterTextSplitter_0-input-separator-string"
+                    }
+                ],
+                "inputAnchors": [],
+                "inputs": {
+                    "chunkSize": "2000",
+                    "chunkOverlap": "200",
+                    "separator": ""
+                },
+                "outputAnchors": [
+                    {
+                        "id": "characterTextSplitter_0-output-characterTextSplitter-CharacterTextSplitter|TextSplitter|BaseDocumentTransformer|Runnable",
+                        "name": "characterTextSplitter",
+                        "label": "CharacterTextSplitter",
+                        "type": "CharacterTextSplitter | TextSplitter | BaseDocumentTransformer | Runnable"
+                    }
+                ],
+                "outputs": {},
+                "selected": false
+            },
+            "selected": false,
+            "positionAbsolute": {
+                "x": 260.5475803279806,
+                "y": -65.1647664861618
+            },
+            "dragging": false
+        },
+        {
+            "width": 300,
+            "height": 332,
+            "id": "openAIEmbeddings_0",
+            "position": {
+                "x": 666.3950526535211,
+                "y": 777.4191705193945
+            },
+            "type": "customNode",
+            "data": {
+                "id": "openAIEmbeddings_0",
+                "label": "OpenAI Embeddings",
+                "version": 1,
+                "name": "openAIEmbeddings",
+                "type": "OpenAIEmbeddings",
+                "baseClasses": ["OpenAIEmbeddings", "Embeddings"],
+                "category": "Embeddings",
+                "description": "OpenAI API to generate embeddings for a given text",
+                "inputParams": [
+                    {
+                        "label": "Connect Credential",
+                        "name": "credential",
+                        "type": "credential",
+                        "credentialNames": ["openAIApi"],
+                        "id": "openAIEmbeddings_0-input-credential-credential"
+                    },
+                    {
+                        "label": "Strip New Lines",
+                        "name": "stripNewLines",
+                        "type": "boolean",
+                        "optional": true,
+                        "additionalParams": true,
+                        "id": "openAIEmbeddings_0-input-stripNewLines-boolean"
+                    },
+                    {
+                        "label": "Batch Size",
+                        "name": "batchSize",
+                        "type": "number",
+                        "optional": true,
+                        "additionalParams": true,
+                        "id": "openAIEmbeddings_0-input-batchSize-number"
+                    },
+                    {
+                        "label": "Timeout",
+                        "name": "timeout",
+                        "type": "number",
+                        "optional": true,
+                        "additionalParams": true,
+                        "id": "openAIEmbeddings_0-input-timeout-number"
+                    },
+                    {
+                        "label": "BasePath",
+                        "name": "basepath",
+                        "type": "string",
+                        "optional": true,
+                        "additionalParams": true,
+                        "id": "openAIEmbeddings_0-input-basepath-string"
+                    }
+                ],
+                "inputAnchors": [],
+                "inputs": {
+                    "stripNewLines": "",
+                    "batchSize": "",
+                    "timeout": "",
+                    "basepath": ""
+                },
+                "outputAnchors": [
+                    {
+                        "id": "openAIEmbeddings_0-output-openAIEmbeddings-OpenAIEmbeddings|Embeddings",
+                        "name": "openAIEmbeddings",
+                        "label": "OpenAIEmbeddings",
+                        "type": "OpenAIEmbeddings | Embeddings"
+                    }
+                ],
+                "outputs": {},
+                "selected": false
+            },
+            "selected": false,
+            "dragging": false,
+            "positionAbsolute": {
+                "x": 666.3950526535211,
+                "y": 777.4191705193945
+            }
+        },
+        {
+            "width": 300,
+            "height": 482,
+            "id": "searchApi_0",
+            "position": {
+                "x": 680.1258121447145,
+                "y": 144.9905217023999
+            },
+            "type": "customNode",
+            "data": {
+                "id": "searchApi_0",
+                "label": "SearchApi",
+                "version": 1,
+                "name": "searchApi",
+                "type": "Document",
+                "baseClasses": ["Document"],
+                "category": "Document Loaders",
+                "description": "Load data from real-time search results",
+                "inputParams": [
+                    {
+                        "label": "Connect Credential",
+                        "name": "credential",
+                        "type": "credential",
+                        "optional": false,
+                        "credentialNames": ["searchApi"],
+                        "id": "searchApi_0-input-credential-credential"
+                    },
+                    {
+                        "label": "Query",
+                        "name": "query",
+                        "type": "string",
+                        "optional": true,
+                        "id": "searchApi_0-input-query-string"
+                    },
+                    {
+                        "label": "Custom Parameters",
+                        "name": "customParameters",
+                        "type": "json",
+                        "optional": true,
+                        "additionalParams": true,
+                        "id": "searchApi_0-input-customParameters-json"
+                    },
+                    {
+                        "label": "Metadata",
+                        "name": "metadata",
+                        "type": "json",
+                        "optional": true,
+                        "additionalParams": true,
+                        "id": "searchApi_0-input-metadata-json"
+                    }
+                ],
+                "inputAnchors": [
+                    {
+                        "label": "Text Splitter",
+                        "name": "textSplitter",
+                        "type": "TextSplitter",
+                        "optional": true,
+                        "id": "searchApi_0-input-textSplitter-TextSplitter"
+                    }
+                ],
+                "inputs": {
+                    "query": "",
+                    "customParameters": "{\"engine\":\"youtube_transcripts\",\"video_id\":\"0e3GPea1Tyg\"}",
+                    "textSplitter": "{{characterTextSplitter_0.data.instance}}",
+                    "metadata": ""
+                },
+                "outputAnchors": [
+                    {
+                        "id": "searchApi_0-output-searchApi-Document",
+                        "name": "searchApi",
+                        "label": "Document",
+                        "type": "Document"
+                    }
+                ],
+                "outputs": {},
+                "selected": false
+            },
+            "selected": false,
+            "positionAbsolute": {
+                "x": 680.1258121447145,
+                "y": 144.9905217023999
+            },
+            "dragging": false
+        }
+    ],
+    "edges": [
+        {
+            "source": "memoryVectorStore_0",
+            "sourceHandle": "memoryVectorStore_0-output-retriever-Memory|VectorStoreRetriever|BaseRetriever",
+            "target": "conversationalRetrievalQAChain_0",
+            "targetHandle": "conversationalRetrievalQAChain_0-input-vectorStoreRetriever-BaseRetriever",
+            "type": "buttonedge",
+            "id": "memoryVectorStore_0-memoryVectorStore_0-output-retriever-Memory|VectorStoreRetriever|BaseRetriever-conversationalRetrievalQAChain_0-conversationalRetrievalQAChain_0-input-vectorStoreRetriever-BaseRetriever",
+            "data": {
+                "label": ""
+            }
+        },
+        {
+            "source": "chatOpenAI_0",
+            "sourceHandle": "chatOpenAI_0-output-chatOpenAI-ChatOpenAI|BaseChatModel|BaseLanguageModel|Runnable",
+            "target": "conversationalRetrievalQAChain_0",
+            "targetHandle": "conversationalRetrievalQAChain_0-input-model-BaseLanguageModel",
+            "type": "buttonedge",
+            "id": "chatOpenAI_0-chatOpenAI_0-output-chatOpenAI-ChatOpenAI|BaseChatModel|BaseLanguageModel|Runnable-conversationalRetrievalQAChain_0-conversationalRetrievalQAChain_0-input-model-BaseLanguageModel",
+            "data": {
+                "label": ""
+            }
+        },
+        {
+            "source": "openAIEmbeddings_0",
+            "sourceHandle": "openAIEmbeddings_0-output-openAIEmbeddings-OpenAIEmbeddings|Embeddings",
+            "target": "memoryVectorStore_0",
+            "targetHandle": "memoryVectorStore_0-input-embeddings-Embeddings",
+            "type": "buttonedge",
+            "id": "openAIEmbeddings_0-openAIEmbeddings_0-output-openAIEmbeddings-OpenAIEmbeddings|Embeddings-memoryVectorStore_0-memoryVectorStore_0-input-embeddings-Embeddings",
+            "data": {
+                "label": ""
+            }
+        },
+        {
+            "source": "characterTextSplitter_0",
+            "sourceHandle": "characterTextSplitter_0-output-characterTextSplitter-CharacterTextSplitter|TextSplitter|BaseDocumentTransformer|Runnable",
+            "target": "searchApi_0",
+            "targetHandle": "searchApi_0-input-textSplitter-TextSplitter",
+            "type": "buttonedge",
+            "id": "characterTextSplitter_0-characterTextSplitter_0-output-characterTextSplitter-CharacterTextSplitter|TextSplitter|BaseDocumentTransformer|Runnable-searchApi_0-searchApi_0-input-textSplitter-TextSplitter",
+            "data": {
+                "label": ""
+            }
+        },
+        {
+            "source": "searchApi_0",
+            "sourceHandle": "searchApi_0-output-searchApi-Document",
+            "target": "memoryVectorStore_0",
+            "targetHandle": "memoryVectorStore_0-input-document-Document",
+            "type": "buttonedge",
+            "id": "searchApi_0-searchApi_0-output-searchApi-Document-memoryVectorStore_0-memoryVectorStore_0-input-document-Document",
+            "data": {
+                "label": ""
+            }
+        }
+    ]
+}

packages/server/marketplaces/chatflows/Conversational Retrieval QA Chain.json

@@ -368,7 +368,7 @@
                 "id": "recursiveCharacterTextSplitter_0",
                 "label": "Recursive Character Text Splitter",
                 "name": "recursiveCharacterTextSplitter",
-                "version": 1,
+                "version": 2,
                 "type": "RecursiveCharacterTextSplitter",
                 "baseClasses": ["RecursiveCharacterTextSplitter", "TextSplitter"],
                 "category": "Text Splitters",
@@ -388,6 +388,17 @@
                         "type": "number",
                         "optional": true,
                         "id": "recursiveCharacterTextSplitter_0-input-chunkOverlap-number"
+                    },
+                    {
+                        "label": "Custom Separators",
+                        "name": "separators",
+                        "type": "string",
+                        "rows": 4,
+                        "description": "Array of custom separators to determine when to split the text, will override the default separators",
+                        "placeholder": "[\"|\", \"##\", \">\", \"-\"]",
+                        "additionalParams": true,
+                        "optional": true,
+                        "id": "recursiveCharacterTextSplitter_0-input-separators-string"
                     }
                 ],
                 "inputAnchors": [],
@@ -426,7 +437,7 @@
                 "id": "textFile_0",
                 "label": "Text File",
                 "name": "textFile",
-                "version": 2,
+                "version": 3,
                 "type": "Document",
                 "baseClasses": ["Document"],
                 "category": "Document Loaders",
@@ -436,7 +447,7 @@
                         "label": "Txt File",
                         "name": "txtFile",
                         "type": "file",
-                        "fileType": ".txt",
+                        "fileType": ".txt, .html, .aspx, .asp, .cpp, .c, .cs, .css, .go, .h, .java, .js, .less, .ts, .php, .proto, .python, .py, .rst, .ruby, .rb, .rs, .scala, .sc, .scss, .sol, .sql, .swift, .markdown, .md, .tex, .ltx, .vb, .xml",
                         "id": "textFile_0-input-txtFile-file"
                     },
                     {

packages/server/marketplaces/chatflows/Local QnA.json

@@ -14,7 +14,7 @@
                 "id": "recursiveCharacterTextSplitter_1",
                 "label": "Recursive Character Text Splitter",
                 "name": "recursiveCharacterTextSplitter",
-                "version": 1,
+                "version": 2,
                 "type": "RecursiveCharacterTextSplitter",
                 "baseClasses": ["RecursiveCharacterTextSplitter", "TextSplitter"],
                 "category": "Text Splitters",
@@ -34,6 +34,17 @@
                         "type": "number",
                         "optional": true,
                         "id": "recursiveCharacterTextSplitter_1-input-chunkOverlap-number"
+                    },
+                    {
+                        "label": "Custom Separators",
+                        "name": "separators",
+                        "type": "string",
+                        "rows": 4,
+                        "description": "Array of custom separators to determine when to split the text, will override the default separators",
+                        "placeholder": "[\"|\", \"##\", \">\", \"-\"]",
+                        "additionalParams": true,
+                        "optional": true,
+                        "id": "recursiveCharacterTextSplitter_1-input-separators-string"
                     }
                 ],
                 "inputAnchors": [],
@@ -373,7 +384,7 @@
                 "id": "textFile_0",
                 "label": "Text File",
                 "name": "textFile",
-                "version": 1,
+                "version": 3,
                 "type": "Document",
                 "baseClasses": ["Document"],
                 "category": "Document Loaders",
@@ -383,7 +394,7 @@
                         "label": "Txt File",
                         "name": "txtFile",
                         "type": "file",
-                        "fileType": ".txt",
+                        "fileType": ".txt, .html, .aspx, .asp, .cpp, .c, .cs, .css, .go, .h, .java, .js, .less, .ts, .php, .proto, .python, .py, .rst, .ruby, .rb, .rs, .scala, .sc, .scss, .sol, .sql, .swift, .markdown, .md, .tex, .ltx, .vb, .xml",
                         "id": "textFile_0-input-txtFile-file"
                     },
                     {

packages/server/marketplaces/chatflows/Metadata Filter Upsert.json

@@ -14,7 +14,7 @@
                 "id": "recursiveCharacterTextSplitter_1",
                 "label": "Recursive Character Text Splitter",
                 "name": "recursiveCharacterTextSplitter",
-                "version": 1,
+                "version": 2,
                 "type": "RecursiveCharacterTextSplitter",
                 "baseClasses": ["RecursiveCharacterTextSplitter", "TextSplitter"],
                 "category": "Text Splitters",
@@ -34,6 +34,17 @@
                         "type": "number",
                         "optional": true,
                         "id": "recursiveCharacterTextSplitter_1-input-chunkOverlap-number"
+                    },
+                    {
+                        "label": "Custom Separators",
+                        "name": "separators",
+                        "type": "string",
+                        "rows": 4,
+                        "description": "Array of custom separators to determine when to split the text, will override the default separators",
+                        "placeholder": "[\"|\", \"##\", \">\", \"-\"]",
+                        "additionalParams": true,
+                        "optional": true,
+                        "id": "recursiveCharacterTextSplitter_1-input-separators-string"
                     }
                 ],
                 "inputAnchors": [],
@@ -72,7 +83,7 @@
                 "id": "textFile_0",
                 "label": "Text File",
                 "name": "textFile",
-                "version": 1,
+                "version": 3,
                 "type": "Document",
                 "baseClasses": ["Document"],
                 "category": "Document Loaders",
@@ -82,7 +93,7 @@
                         "label": "Txt File",
                         "name": "txtFile",
                         "type": "file",
-                        "fileType": ".txt",
+                        "fileType": ".txt, .html, .aspx, .asp, .cpp, .c, .cs, .css, .go, .h, .java, .js, .less, .ts, .php, .proto, .python, .py, .rst, .ruby, .rb, .rs, .scala, .sc, .scss, .sol, .sql, .swift, .markdown, .md, .tex, .ltx, .vb, .xml",
                         "id": "textFile_0-input-txtFile-file"
                     },
                     {

packages/server/marketplaces/chatflows/Prompt Chaining with VectorStore.json

@@ -3,120 +3,11 @@
     "nodes": [
         {
             "width": 300,
-            "height": 503,
+            "height": 329,
-            "id": "pineconeExistingIndex_0",
-            "position": {
-                "x": 1062.7418678410986,
-                "y": -109.27680365777141
-            },
-            "type": "customNode",
-            "data": {
-                "id": "pineconeExistingIndex_0",
-                "label": "Pinecone Load Existing Index",
-                "version": 1,
-                "name": "pineconeExistingIndex",
-                "type": "Pinecone",
-                "baseClasses": ["Pinecone", "VectorStoreRetriever", "BaseRetriever"],
-                "category": "Vector Stores",
-                "description": "Load existing index from Pinecone (i.e: Document has been upserted)",
-                "inputParams": [
-                    {
-                        "label": "Connect Credential",
-                        "name": "credential",
-                        "type": "credential",
-                        "credentialNames": ["pineconeApi"],
-                        "id": "pineconeExistingIndex_0-input-credential-credential"
-                    },
-                    {
-                        "label": "Pinecone Index",
-                        "name": "pineconeIndex",
-                        "type": "string",
-                        "id": "pineconeExistingIndex_0-input-pineconeIndex-string"
-                    },
-                    {
-                        "label": "Pinecone Namespace",
-                        "name": "pineconeNamespace",
-                        "type": "string",
-                        "placeholder": "my-first-namespace",
-                        "additionalParams": true,
-                        "optional": true,
-                        "id": "pineconeExistingIndex_0-input-pineconeNamespace-string"
-                    },
-                    {
-                        "label": "Pinecone Metadata Filter",
-                        "name": "pineconeMetadataFilter",
-                        "type": "json",
-                        "optional": true,
-                        "additionalParams": true,
-                        "id": "pineconeExistingIndex_0-input-pineconeMetadataFilter-json"
-                    },
-                    {
-                        "label": "Top K",
-                        "name": "topK",
-                        "description": "Number of top results to fetch. Default to 4",
-                        "placeholder": "4",
-                        "type": "number",
-                        "additionalParams": true,
-                        "optional": true,
-                        "id": "pineconeExistingIndex_0-input-topK-number"
-                    }
-                ],
-                "inputAnchors": [
-                    {
-                        "label": "Embeddings",
-                        "name": "embeddings",
-                        "type": "Embeddings",
-                        "id": "pineconeExistingIndex_0-input-embeddings-Embeddings"
-                    }
-                ],
-                "inputs": {
-                    "embeddings": "{{openAIEmbeddings_0.data.instance}}",
-                    "pineconeIndex": "newindex",
-                    "pineconeNamespace": "",
-                    "pineconeMetadataFilter": "{}",
-                    "topK": ""
-                },
-                "outputAnchors": [
-                    {
-                        "name": "output",
-                        "label": "Output",
-                        "type": "options",
-                        "options": [
-                            {
-                                "id": "pineconeExistingIndex_0-output-retriever-Pinecone|VectorStoreRetriever|BaseRetriever",
-                                "name": "retriever",
-                                "label": "Pinecone Retriever",
-                                "type": "Pinecone | VectorStoreRetriever | BaseRetriever"
-                            },
-                            {
-                                "id": "pineconeExistingIndex_0-output-vectorStore-Pinecone|VectorStore",
-                                "name": "vectorStore",
-                                "label": "Pinecone Vector Store",
-                                "type": "Pinecone | VectorStore"
-                            }
-                        ],
-                        "default": "retriever"
-                    }
-                ],
-                "outputs": {
-                    "output": "vectorStore"
-                },
-                "selected": false
-            },
-            "selected": false,
-            "positionAbsolute": {
-                "x": 1062.7418678410986,
-                "y": -109.27680365777141
-            },
-            "dragging": false
-        },
-        {
-            "width": 300,
-            "height": 327,
             "id": "openAIEmbeddings_0",
             "position": {
-                "x": 711.3971966563331,
+                "x": 1198.6643452533754,
-                "y": 7.7184225021727
+                "y": -584.4233173804803
             },
             "type": "customNode",
             "data": {
@@ -189,18 +80,18 @@
             },
             "selected": false,
             "positionAbsolute": {
-                "x": 711.3971966563331,
+                "x": 1198.6643452533754,
-                "y": 7.7184225021727
+                "y": -584.4233173804803
             },
             "dragging": false
         },
         {
             "width": 300,
-            "height": 473,
+            "height": 475,
             "id": "promptTemplate_0",
             "position": {
-                "x": 348.2881107399286,
+                "x": 354.2706973608643,
-                "y": -97.74510214137423
+                "y": -122.34815000085804
             },
             "type": "customNode",
             "data": {
@@ -249,18 +140,18 @@
             },
             "selected": false,
             "positionAbsolute": {
-                "x": 348.2881107399286,
+                "x": 354.2706973608643,
-                "y": -97.74510214137423
+                "y": -122.34815000085804
             },
             "dragging": false
         },
         {
             "width": 300,
-            "height": 522,
+            "height": 574,
             "id": "chatOpenAI_0",
             "position": {
-                "x": 335.7621848973805,
+                "x": 353.5672832154869,
-                "y": -721.7411273245009
+                "y": -730.6436764835541
             },
             "type": "customNode",
             "data": {
@@ -396,7 +287,7 @@
                 ],
                 "inputs": {
                     "modelName": "gpt-3.5-turbo-16k",
-                    "temperature": 0.9,
+                    "temperature": "0",
                     "maxTokens": "",
                     "topP": "",
                     "frequencyPenalty": "",
@@ -418,17 +309,17 @@
             "selected": false,
             "dragging": false,
             "positionAbsolute": {
-                "x": 335.7621848973805,
+                "x": 353.5672832154869,
-                "y": -721.7411273245009
+                "y": -730.6436764835541
             }
         },
         {
             "width": 300,
-            "height": 522,
+            "height": 574,
             "id": "chatOpenAI_1",
             "position": {
-                "x": 1765.2801848172305,
+                "x": 2281.9246645710673,
-                "y": -737.9261054149061
+                "y": -778.8379360672121
             },
             "type": "customNode",
             "data": {
@@ -564,7 +455,7 @@
                 ],
                 "inputs": {
                     "modelName": "gpt-3.5-turbo-16k",
-                    "temperature": 0.9,
+                    "temperature": "0",
                     "maxTokens": "",
                     "topP": "",
                     "frequencyPenalty": "",
@@ -586,36 +477,153 @@
             "selected": false,
             "dragging": false,
             "positionAbsolute": {
-                "x": 1765.2801848172305,
+                "x": 2281.9246645710673,
-                "y": -737.9261054149061
+                "y": -778.8379360672121
             }
         },
         {
             "width": 300,
-            "height": 473,
+            "height": 505,
-            "id": "promptTemplate_1",
+            "id": "pineconeExistingIndex_0",
             "position": {
-                "x": 1773.720934090435,
+                "x": 1544.4998097474581,
-                "y": -116.71323227575395
+                "y": -628.8477510577202
             },
             "type": "customNode",
             "data": {
-                "id": "promptTemplate_1",
+                "id": "pineconeExistingIndex_0",
-                "label": "Prompt Template",
+                "label": "Pinecone Load Existing Index",
                 "version": 1,
-                "name": "promptTemplate",
+                "name": "pineconeExistingIndex",
-                "type": "PromptTemplate",
+                "type": "Pinecone",
-                "baseClasses": ["PromptTemplate", "BaseStringPromptTemplate", "BasePromptTemplate", "Runnable"],
+                "baseClasses": ["Pinecone", "VectorStoreRetriever", "BaseRetriever"],
-                "category": "Prompts",
+                "category": "Vector Stores",
-                "description": "Schema to represent a basic prompt for an LLM",
+                "description": "Load existing index from Pinecone (i.e: Document has been upserted)",
                 "inputParams": [
                     {
-                        "label": "Template",
+                        "label": "Connect Credential",
-                        "name": "template",
+                        "name": "credential",
+                        "type": "credential",
+                        "credentialNames": ["pineconeApi"],
+                        "id": "pineconeExistingIndex_0-input-credential-credential"
+                    },
+                    {
+                        "label": "Pinecone Index",
+                        "name": "pineconeIndex",
+                        "type": "string",
+                        "id": "pineconeExistingIndex_0-input-pineconeIndex-string"
+                    },
+                    {
+                        "label": "Pinecone Namespace",
+                        "name": "pineconeNamespace",
+                        "type": "string",
+                        "placeholder": "my-first-namespace",
+                        "additionalParams": true,
+                        "optional": true,
+                        "id": "pineconeExistingIndex_0-input-pineconeNamespace-string"
+                    },
+                    {
+                        "label": "Pinecone Metadata Filter",
+                        "name": "pineconeMetadataFilter",
+                        "type": "json",
+                        "optional": true,
+                        "additionalParams": true,
+                        "id": "pineconeExistingIndex_0-input-pineconeMetadataFilter-json"
+                    },
+                    {
+                        "label": "Top K",
+                        "name": "topK",
+                        "description": "Number of top results to fetch. Default to 4",
+                        "placeholder": "4",
+                        "type": "number",
+                        "additionalParams": true,
+                        "optional": true,
+                        "id": "pineconeExistingIndex_0-input-topK-number"
+                    }
+                ],
+                "inputAnchors": [
+                    {
+                        "label": "Embeddings",
+                        "name": "embeddings",
+                        "type": "Embeddings",
+                        "id": "pineconeExistingIndex_0-input-embeddings-Embeddings"
+                    }
+                ],
+                "inputs": {
+                    "embeddings": "{{openAIEmbeddings_0.data.instance}}",
+                    "pineconeIndex": "",
+                    "pineconeNamespace": "",
+                    "pineconeMetadataFilter": "",
+                    "topK": ""
+                },
+                "outputAnchors": [
+                    {
+                        "name": "output",
+                        "label": "Output",
+                        "type": "options",
+                        "options": [
+                            {
+                                "id": "pineconeExistingIndex_0-output-retriever-Pinecone|VectorStoreRetriever|BaseRetriever",
+                                "name": "retriever",
+                                "label": "Pinecone Retriever",
+                                "type": "Pinecone | VectorStoreRetriever | BaseRetriever"
+                            },
+                            {
+                                "id": "pineconeExistingIndex_0-output-vectorStore-Pinecone|VectorStore",
+                                "name": "vectorStore",
+                                "label": "Pinecone Vector Store",
+                                "type": "Pinecone | VectorStore"
+                            }
+                        ],
+                        "default": "retriever"
+                    }
+                ],
+                "outputs": {
+                    "output": "vectorStore"
+                },
+                "selected": false
+            },
+            "selected": false,
+            "positionAbsolute": {
+                "x": 1544.4998097474581,
+                "y": -628.8477510577202
+            },
+            "dragging": false
+        },
+        {
+            "width": 300,
+            "height": 652,
+            "id": "chatPromptTemplate_0",
+            "position": {
+                "x": 2290.8365353040026,
+                "y": -168.49082887954518
+            },
+            "type": "customNode",
+            "data": {
+                "id": "chatPromptTemplate_0",
+                "label": "Chat Prompt Template",
+                "version": 1,
+                "name": "chatPromptTemplate",
+                "type": "ChatPromptTemplate",
+                "baseClasses": ["ChatPromptTemplate", "BaseChatPromptTemplate", "BasePromptTemplate", "Runnable"],
+                "category": "Prompts",
+                "description": "Schema to represent a chat prompt",
+                "inputParams": [
+                    {
+                        "label": "System Message",
+                        "name": "systemMessagePrompt",
                         "type": "string",
                         "rows": 4,
-                        "placeholder": "What is a good name for a company that makes {product}?",
+                        "placeholder": "You are a helpful assistant that translates {input_language} to {output_language}.",
-                        "id": "promptTemplate_1-input-template-string"
+                        "id": "chatPromptTemplate_0-input-systemMessagePrompt-string"
+                    },
+                    {
+                        "label": "Human Message",
+                        "name": "humanMessagePrompt",
+                        "type": "string",
+                        "rows": 4,
+                        "placeholder": "{text}",
+                        "id": "chatPromptTemplate_0-input-humanMessagePrompt-string"
                     },
                     {
                         "label": "Format Prompt Values",
@@ -624,20 +632,21 @@
                         "optional": true,
                         "acceptVariable": true,
                         "list": true,
-                        "id": "promptTemplate_1-input-promptValues-json"
+                        "id": "chatPromptTemplate_0-input-promptValues-json"
                     }
                 ],
                 "inputAnchors": [],
                 "inputs": {
-                    "template": "Use the following pieces of context to answer the question at the end.\n\n{context}\n\nQuestion: {question}\nHelpful Answer:",
+                    "systemMessagePrompt": "Using the provided context, answer the user's question to the best of your ability using the resources provided. If there is nothing in the context relevant to the question at hand, just say \"Hmm, I'm not sure.\" Don't try to make up an answer.\n\nAnything between the following \\`context\\`  html blocks is retrieved from a knowledge bank, not part of the conversation with the user.\n\n<context>\n    {context}\n<context/>\n\nREMEMBER: If there is no relevant information within the context, just say \"Hmm, I'm not sure.\" Don't try to make up an answer. Anything between the preceding 'context' html blocks is retrieved from a knowledge bank, not part of the conversation with the user.",
-                    "promptValues": "{\"context\":\"{{vectorStoreToDocument_0.data.instance}}\",\"question\":\"{{llmChain_0.data.instance}}\"}"
+                    "humanMessagePrompt": "{text}",
+                    "promptValues": "{\"context\":\"{{vectorStoreToDocument_0.data.instance}}\",\"text\":\"{{question}}\"}"
                 },
                 "outputAnchors": [
                     {
-                        "id": "promptTemplate_1-output-promptTemplate-PromptTemplate|BaseStringPromptTemplate|BasePromptTemplate|Runnable",
+                        "id": "chatPromptTemplate_0-output-chatPromptTemplate-ChatPromptTemplate|BaseChatPromptTemplate|BasePromptTemplate|Runnable",
-                        "name": "promptTemplate",
+                        "name": "chatPromptTemplate",
-                        "label": "PromptTemplate",
+                        "label": "ChatPromptTemplate",
-                        "type": "PromptTemplate | BaseStringPromptTemplate | BasePromptTemplate | Runnable"
+                        "type": "ChatPromptTemplate | BaseChatPromptTemplate | BasePromptTemplate | Runnable"
                     }
                 ],
                 "outputs": {},
@@ -645,18 +654,18 @@
             },
             "selected": false,
             "positionAbsolute": {
-                "x": 1773.720934090435,
+                "x": 2290.8365353040026,
-                "y": -116.71323227575395
+                "y": -168.49082887954518
             },
             "dragging": false
         },
         {
             "width": 300,
-            "height": 404,
+            "height": 405,
             "id": "llmChain_0",
             "position": {
-                "x": 756.1670091985342,
+                "x": 747.1299875516488,
-                "y": -592.5151355056942
+                "y": -267.01184813798244
             },
             "type": "customNode",
             "data": {
@@ -695,7 +704,7 @@
                 "inputs": {
                     "model": "{{chatOpenAI_0.data.instance}}",
                     "prompt": "{{promptTemplate_0.data.instance}}",
-                    "chainName": "QuestionChain"
+                    "chainName": "RephraseQuestion"
                 },
                 "outputAnchors": [
                     {
@@ -726,18 +735,18 @@
             },
             "selected": false,
             "positionAbsolute": {
-                "x": 756.1670091985342,
+                "x": 747.1299875516488,
-                "y": -592.5151355056942
+                "y": -267.01184813798244
             },
             "dragging": false
         },
         {
             "width": 300,
-            "height": 404,
+            "height": 405,
             "id": "llmChain_1",
             "position": {
-                "x": 2200.1274896215496,
+                "x": 2694.8707655351186,
-                "y": -144.29167974642334
+                "y": -308.59150355411236
             },
             "type": "customNode",
             "data": {
@@ -775,8 +784,8 @@
                 ],
                 "inputs": {
                     "model": "{{chatOpenAI_1.data.instance}}",
-                    "prompt": "{{promptTemplate_1.data.instance}}",
+                    "prompt": "{{chatPromptTemplate_0.data.instance}}",
-                    "chainName": ""
+                    "chainName": "FinalResponse"
                 },
                 "outputAnchors": [
                     {
@@ -807,30 +816,39 @@
             },
             "selected": false,
             "positionAbsolute": {
-                "x": 2200.1274896215496,
+                "x": 2694.8707655351186,
-                "y": -144.29167974642334
+                "y": -308.59150355411236
             },
             "dragging": false
         },
         {
             "width": 300,
-            "height": 353,
+            "height": 454,
             "id": "vectorStoreToDocument_0",
             "position": {
-                "x": 1407.7038120189868,
+                "x": 1906.6871314089658,
-                "y": -26.16468811205081
+                "y": -157.0046189166955
             },
             "type": "customNode",
             "data": {
                 "id": "vectorStoreToDocument_0",
                 "label": "VectorStore To Document",
-                "version": 1,
+                "version": 2,
                 "name": "vectorStoreToDocument",
                 "type": "Document",
                 "baseClasses": ["Document"],
                 "category": "Document Loaders",
                 "description": "Search documents with scores from vector store",
                 "inputParams": [
+                    {
+                        "label": "Query",
+                        "name": "query",
+                        "type": "string",
+                        "description": "Query to retrieve documents from vector database. If not specified, user question will be used",
+                        "optional": true,
+                        "acceptVariable": true,
+                        "id": "vectorStoreToDocument_0-input-query-string"
+                    },
                     {
                         "label": "Minimum Score (%)",
                         "name": "minScore",
@@ -852,6 +870,7 @@
                 ],
                 "inputs": {
                     "vectorStore": "{{pineconeExistingIndex_0.data.instance}}",
+                    "query": "{{llmChain_0.data.instance}}",
                     "minScore": ""
                 },
                 "outputAnchors": [
@@ -883,8 +902,8 @@
             },
             "selected": false,
             "positionAbsolute": {
-                "x": 1407.7038120189868,
+                "x": 1906.6871314089658,
-                "y": -26.16468811205081
+                "y": -157.0046189166955
             },
             "dragging": false
         }
@@ -901,6 +920,50 @@
                 "label": ""
             }
         },
+        {
+            "source": "promptTemplate_0",
+            "sourceHandle": "promptTemplate_0-output-promptTemplate-PromptTemplate|BaseStringPromptTemplate|BasePromptTemplate|Runnable",
+            "target": "llmChain_0",
+            "targetHandle": "llmChain_0-input-prompt-BasePromptTemplate",
+            "type": "buttonedge",
+            "id": "promptTemplate_0-promptTemplate_0-output-promptTemplate-PromptTemplate|BaseStringPromptTemplate|BasePromptTemplate|Runnable-llmChain_0-llmChain_0-input-prompt-BasePromptTemplate",
+            "data": {
+                "label": ""
+            }
+        },
+        {
+            "source": "chatOpenAI_0",
+            "sourceHandle": "chatOpenAI_0-output-chatOpenAI-ChatOpenAI|BaseChatModel|BaseLanguageModel|Runnable",
+            "target": "llmChain_0",
+            "targetHandle": "llmChain_0-input-model-BaseLanguageModel",
+            "type": "buttonedge",
+            "id": "chatOpenAI_0-chatOpenAI_0-output-chatOpenAI-ChatOpenAI|BaseChatModel|BaseLanguageModel|Runnable-llmChain_0-llmChain_0-input-model-BaseLanguageModel",
+            "data": {
+                "label": ""
+            }
+        },
+        {
+            "source": "chatPromptTemplate_0",
+            "sourceHandle": "chatPromptTemplate_0-output-chatPromptTemplate-ChatPromptTemplate|BaseChatPromptTemplate|BasePromptTemplate|Runnable",
+            "target": "llmChain_1",
+            "targetHandle": "llmChain_1-input-prompt-BasePromptTemplate",
+            "type": "buttonedge",
+            "id": "chatPromptTemplate_0-chatPromptTemplate_0-output-chatPromptTemplate-ChatPromptTemplate|BaseChatPromptTemplate|BasePromptTemplate|Runnable-llmChain_1-llmChain_1-input-prompt-BasePromptTemplate",
+            "data": {
+                "label": ""
+            }
+        },
+        {
+            "source": "chatOpenAI_1",
+            "sourceHandle": "chatOpenAI_1-output-chatOpenAI-ChatOpenAI|BaseChatModel|BaseLanguageModel|Runnable",
+            "target": "llmChain_1",
+            "targetHandle": "llmChain_1-input-model-BaseLanguageModel",
+            "type": "buttonedge",
+            "id": "chatOpenAI_1-chatOpenAI_1-output-chatOpenAI-ChatOpenAI|BaseChatModel|BaseLanguageModel|Runnable-llmChain_1-llmChain_1-input-model-BaseLanguageModel",
+            "data": {
+                "label": ""
+            }
+        },
         {
             "source": "pineconeExistingIndex_0",
             "sourceHandle": "pineconeExistingIndex_0-output-vectorStore-Pinecone|VectorStore",
@@ -915,32 +978,10 @@
         {
             "source": "vectorStoreToDocument_0",
             "sourceHandle": "vectorStoreToDocument_0-output-text-string|json",
-            "target": "promptTemplate_1",
+            "target": "chatPromptTemplate_0",
-            "targetHandle": "promptTemplate_1-input-promptValues-json",
+            "targetHandle": "chatPromptTemplate_0-input-promptValues-json",
             "type": "buttonedge",
-            "id": "vectorStoreToDocument_0-vectorStoreToDocument_0-output-text-string|json-promptTemplate_1-promptTemplate_1-input-promptValues-json",
+            "id": "vectorStoreToDocument_0-vectorStoreToDocument_0-output-text-string|json-chatPromptTemplate_0-chatPromptTemplate_0-input-promptValues-json",
-            "data": {
-                "label": ""
-            }
-        },
-        {
-            "source": "chatOpenAI_0",
-            "sourceHandle": "chatOpenAI_0-output-chatOpenAI-ChatOpenAI|BaseChatModel|BaseLanguageModel|Runnable",
-            "target": "llmChain_0",
-            "targetHandle": "llmChain_0-input-model-BaseLanguageModel",
-            "type": "buttonedge",
-            "id": "chatOpenAI_0-chatOpenAI_0-output-chatOpenAI-ChatOpenAI|BaseChatModel|BaseLanguageModel|Runnable-llmChain_0-llmChain_0-input-model-BaseLanguageModel",
-            "data": {
-                "label": ""
-            }
-        },
-        {
-            "source": "promptTemplate_0",
-            "sourceHandle": "promptTemplate_0-output-promptTemplate-PromptTemplate|BaseStringPromptTemplate|BasePromptTemplate|Runnable",
-            "target": "llmChain_0",
-            "targetHandle": "llmChain_0-input-prompt-BasePromptTemplate",
-            "type": "buttonedge",
-            "id": "promptTemplate_0-promptTemplate_0-output-promptTemplate-PromptTemplate|BaseStringPromptTemplate|BasePromptTemplate|Runnable-llmChain_0-llmChain_0-input-prompt-BasePromptTemplate",
             "data": {
                 "label": ""
             }
@@ -948,32 +989,10 @@
         {
             "source": "llmChain_0",
             "sourceHandle": "llmChain_0-output-outputPrediction-string|json",
-            "target": "promptTemplate_1",
+            "target": "vectorStoreToDocument_0",
-            "targetHandle": "promptTemplate_1-input-promptValues-json",
+            "targetHandle": "vectorStoreToDocument_0-input-query-string",
             "type": "buttonedge",
-            "id": "llmChain_0-llmChain_0-output-outputPrediction-string|json-promptTemplate_1-promptTemplate_1-input-promptValues-json",
+            "id": "llmChain_0-llmChain_0-output-outputPrediction-string|json-vectorStoreToDocument_0-vectorStoreToDocument_0-input-query-string",
-            "data": {
-                "label": ""
-            }
-        },
-        {
-            "source": "chatOpenAI_1",
-            "sourceHandle": "chatOpenAI_1-output-chatOpenAI-ChatOpenAI|BaseChatModel|BaseLanguageModel|Runnable",
-            "target": "llmChain_1",
-            "targetHandle": "llmChain_1-input-model-BaseLanguageModel",
-            "type": "buttonedge",
-            "id": "chatOpenAI_1-chatOpenAI_1-output-chatOpenAI-ChatOpenAI|BaseChatModel|BaseLanguageModel|Runnable-llmChain_1-llmChain_1-input-model-BaseLanguageModel",
-            "data": {
-                "label": ""
-            }
-        },
-        {
-            "source": "promptTemplate_1",
-            "sourceHandle": "promptTemplate_1-output-promptTemplate-PromptTemplate|BaseStringPromptTemplate|BasePromptTemplate|Runnable",
-            "target": "llmChain_1",
-            "targetHandle": "llmChain_1-input-prompt-BasePromptTemplate",
-            "type": "buttonedge",
-            "id": "promptTemplate_1-promptTemplate_1-output-promptTemplate-PromptTemplate|BaseStringPromptTemplate|BasePromptTemplate|Runnable-llmChain_1-llmChain_1-input-prompt-BasePromptTemplate",
             "data": {
                 "label": ""
             }

packages/server/package.json

@@ -64,6 +64,7 @@
         "socket.io": "^4.6.1",
         "sqlite3": "^5.1.6",
         "typeorm": "^0.3.6",
+        "uuid": "^9.0.1",
         "winston": "^3.9.0"
     },
     "devDependencies": {

packages/server/src/Interface.ts

@@ -2,6 +2,10 @@ import { ICommonObject, INode, INodeData as INodeDataFromComponent, INodeParams
 
 export type MessageType = 'apiMessage' | 'userMessage'
 
+export enum chatType {
+    INTERNAL = 'INTERNAL',
+    EXTERNAL = 'EXTERNAL'
+}
 /**
  * Databases
  */
@@ -24,8 +28,12 @@ export interface IChatMessage {
     role: MessageType
     content: string
     chatflowid: string
-    createdDate: Date
     sourceDocuments?: string
+    chatType: string
+    chatId: string
+    memoryType?: string
+    sessionId?: string
+    createdDate: Date
 }
 
 export interface ITool {
@@ -146,6 +154,7 @@ export interface IncomingInput {
     history: IMessage[]
     overrideConfig?: ICommonObject
     socketIOClientId?: string
+    chatId?: string
 }
 
 export interface IActiveChatflows {

packages/server/src/database/entities/ChatMessage.ts

@@ -20,6 +20,18 @@ export class ChatMessage implements IChatMessage {
     @Column({ nullable: true, type: 'text' })
     sourceDocuments?: string
 
+    @Column()
+    chatType: string
+
+    @Column()
+    chatId: string
+
+    @Column({ nullable: true })
+    memoryType?: string
+
+    @Column({ nullable: true })
+    sessionId?: string
+
     @CreateDateColumn()
     createdDate: Date
 }

packages/server/src/database/migrations/mysql/1694099200729-AddApiConfig.ts

@@ -2,7 +2,8 @@ import { MigrationInterface, QueryRunner } from 'typeorm'
 
 export class AddApiConfig1694099200729 implements MigrationInterface {
     public async up(queryRunner: QueryRunner): Promise<void> {
-        await queryRunner.query(`ALTER TABLE \`chat_flow\` ADD COLUMN \`apiConfig\` TEXT;`)
+        const columnExists = await queryRunner.hasColumn('chat_flow', 'apiConfig')
+        if (!columnExists) queryRunner.query(`ALTER TABLE \`chat_flow\` ADD COLUMN \`apiConfig\` TEXT;`)
     }
 
     public async down(queryRunner: QueryRunner): Promise<void> {

packages/server/src/database/migrations/mysql/1694432361423-AddAnalytic.ts

@@ -2,7 +2,8 @@ import { MigrationInterface, QueryRunner } from 'typeorm'
 
 export class AddAnalytic1694432361423 implements MigrationInterface {
     public async up(queryRunner: QueryRunner): Promise<void> {
-        await queryRunner.query(`ALTER TABLE \`chat_flow\` ADD COLUMN \`analytic\` TEXT;`)
+        const columnExists = await queryRunner.hasColumn('chat_flow', 'analytic')
+        if (!columnExists) queryRunner.query(`ALTER TABLE \`chat_flow\` ADD COLUMN \`analytic\` TEXT;`)
     }
 
     public async down(queryRunner: QueryRunner): Promise<void> {

packages/server/src/database/migrations/mysql/1694658767766-AddChatHistory.ts

@@ -0,0 +1,41 @@
+import { MigrationInterface, QueryRunner } from 'typeorm'
+
+export class AddChatHistory1694658767766 implements MigrationInterface {
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        const chatTypeColumnExists = await queryRunner.hasColumn('chat_message', 'chatType')
+        if (!chatTypeColumnExists)
+            await queryRunner.query(`ALTER TABLE \`chat_message\` ADD COLUMN \`chatType\` VARCHAR(255) NOT NULL DEFAULT 'INTERNAL';`)
+
+        const chatIdColumnExists = await queryRunner.hasColumn('chat_message', 'chatId')
+        if (!chatIdColumnExists) await queryRunner.query(`ALTER TABLE \`chat_message\` ADD COLUMN \`chatId\` VARCHAR(255);`)
+        const results: { id: string; chatflowid: string }[] = await queryRunner.query(`WITH RankedMessages AS (
+                SELECT
+                    \`chatflowid\`,
+                    \`id\`,
+                    \`createdDate\`,
+                    ROW_NUMBER() OVER (PARTITION BY \`chatflowid\` ORDER BY \`createdDate\`) AS row_num
+                FROM \`chat_message\`
+            )
+            SELECT \`chatflowid\`, \`id\`
+            FROM RankedMessages
+            WHERE row_num = 1;`)
+        for (const chatMessage of results) {
+            await queryRunner.query(
+                `UPDATE \`chat_message\` SET \`chatId\` = '${chatMessage.id}' WHERE \`chatflowid\` = '${chatMessage.chatflowid}'`
+            )
+        }
+        await queryRunner.query(`ALTER TABLE \`chat_message\` MODIFY \`chatId\` VARCHAR(255) NOT NULL;`)
+
+        const memoryTypeColumnExists = await queryRunner.hasColumn('chat_message', 'memoryType')
+        if (!memoryTypeColumnExists) await queryRunner.query(`ALTER TABLE \`chat_message\` ADD COLUMN \`memoryType\` VARCHAR(255);`)
+
+        const sessionIdColumnExists = await queryRunner.hasColumn('chat_message', 'sessionId')
+        if (!sessionIdColumnExists) await queryRunner.query(`ALTER TABLE \`chat_message\` ADD COLUMN \`sessionId\` VARCHAR(255);`)
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(
+            `ALTER TABLE \`chat_message\` DROP COLUMN \`chatType\`, DROP COLUMN \`chatId\`, DROP COLUMN \`memoryType\`, DROP COLUMN \`sessionId\`;`
+        )
+    }
+}

packages/server/src/database/migrations/mysql/index.ts

@@ -5,6 +5,7 @@ import { ModifyCredential1693999261583 } from './1693999261583-ModifyCredential'
 import { ModifyTool1694001465232 } from './1694001465232-ModifyTool'
 import { AddApiConfig1694099200729 } from './1694099200729-AddApiConfig'
 import { AddAnalytic1694432361423 } from './1694432361423-AddAnalytic'
+import { AddChatHistory1694658767766 } from './1694658767766-AddChatHistory'
 
 export const mysqlMigrations = [
     Init1693840429259,
@@ -13,5 +14,6 @@ export const mysqlMigrations = [
     ModifyCredential1693999261583,
     ModifyTool1694001465232,
     AddApiConfig1694099200729,
-    AddAnalytic1694432361423
+    AddAnalytic1694432361423,
+    AddChatHistory1694658767766
 ]

packages/server/src/database/migrations/postgres/1694099183389-AddApiConfig.ts

@@ -2,7 +2,7 @@ import { MigrationInterface, QueryRunner } from 'typeorm'
 
 export class AddApiConfig1694099183389 implements MigrationInterface {
     public async up(queryRunner: QueryRunner): Promise<void> {
-        await queryRunner.query(`ALTER TABLE "chat_flow" ADD COLUMN "apiConfig" TEXT;`)
+        await queryRunner.query(`ALTER TABLE "chat_flow" ADD COLUMN IF NOT EXISTS "apiConfig" TEXT;`)
     }
 
     public async down(queryRunner: QueryRunner): Promise<void> {

packages/server/src/database/migrations/postgres/1694432361423-AddAnalytic.ts

@@ -2,7 +2,7 @@ import { MigrationInterface, QueryRunner } from 'typeorm'
 
 export class AddAnalytic1694432361423 implements MigrationInterface {
     public async up(queryRunner: QueryRunner): Promise<void> {
-        await queryRunner.query(`ALTER TABLE "chat_flow" ADD COLUMN "analytic" TEXT;`)
+        await queryRunner.query(`ALTER TABLE "chat_flow" ADD COLUMN IF NOT EXISTS "analytic" TEXT;`)
     }
 
     public async down(queryRunner: QueryRunner): Promise<void> {

packages/server/src/database/migrations/postgres/1694658756136-AddChatHistory.ts

@@ -0,0 +1,32 @@
+import { MigrationInterface, QueryRunner } from 'typeorm'
+
+export class AddChatHistory1694658756136 implements MigrationInterface {
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(
+            `ALTER TABLE "chat_message" ADD COLUMN IF NOT EXISTS "chatType" VARCHAR NOT NULL DEFAULT 'INTERNAL', ADD COLUMN IF NOT EXISTS "chatId" VARCHAR, ADD COLUMN IF NOT EXISTS "memoryType" VARCHAR, ADD COLUMN IF NOT EXISTS "sessionId" VARCHAR;`
+        )
+        const results: { id: string; chatflowid: string }[] = await queryRunner.query(`WITH RankedMessages AS (
+                SELECT
+                    "chatflowid",
+                    "id",
+                    "createdDate",
+                    ROW_NUMBER() OVER (PARTITION BY "chatflowid" ORDER BY "createdDate") AS row_num
+                FROM "chat_message"
+            )
+            SELECT "chatflowid", "id"
+            FROM RankedMessages
+            WHERE row_num = 1;`)
+        for (const chatMessage of results) {
+            await queryRunner.query(
+                `UPDATE "chat_message" SET "chatId" = '${chatMessage.id}' WHERE "chatflowid" = '${chatMessage.chatflowid}'`
+            )
+        }
+        await queryRunner.query(`ALTER TABLE "chat_message" ALTER COLUMN "chatId" SET NOT NULL;`)
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(
+            `ALTER TABLE "chat_message" DROP COLUMN "chatType", DROP COLUMN "chatId", DROP COLUMN "memoryType", DROP COLUMN "sessionId";`
+        )
+    }
+}

packages/server/src/database/migrations/postgres/index.ts

@@ -5,6 +5,7 @@ import { ModifyCredential1693997070000 } from './1693997070000-ModifyCredential'
 import { ModifyTool1693997339912 } from './1693997339912-ModifyTool'
 import { AddApiConfig1694099183389 } from './1694099183389-AddApiConfig'
 import { AddAnalytic1694432361423 } from './1694432361423-AddAnalytic'
+import { AddChatHistory1694658756136 } from './1694658756136-AddChatHistory'
 
 export const postgresMigrations = [
     Init1693891895163,
@@ -13,5 +14,6 @@ export const postgresMigrations = [
     ModifyCredential1693997070000,
     ModifyTool1693997339912,
     AddApiConfig1694099183389,
-    AddAnalytic1694432361423
+    AddAnalytic1694432361423,
+    AddChatHistory1694658756136
 ]

packages/server/src/database/migrations/sqlite/1694657778173-AddChatHistory.ts

@@ -0,0 +1,40 @@
+import { MigrationInterface, QueryRunner } from 'typeorm'
+
+export class AddChatHistory1694657778173 implements MigrationInterface {
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE "chat_message" ADD COLUMN "chatId" VARCHAR;`)
+        const results: { id: string; chatflowid: string }[] = await queryRunner.query(`WITH RankedMessages AS (
+                SELECT
+                    "chatflowid",
+                    "id",
+                    "createdDate",
+                    ROW_NUMBER() OVER (PARTITION BY "chatflowid" ORDER BY "createdDate") AS row_num
+                FROM "chat_message"
+            )
+            SELECT "chatflowid", "id"
+            FROM RankedMessages
+            WHERE row_num = 1;`)
+        for (const chatMessage of results) {
+            await queryRunner.query(
+                `UPDATE "chat_message" SET "chatId" = '${chatMessage.id}' WHERE "chatflowid" = '${chatMessage.chatflowid}'`
+            )
+        }
+        await queryRunner.query(
+            `CREATE TABLE "temp_chat_message" ("id" varchar PRIMARY KEY NOT NULL, "role" varchar NOT NULL, "chatflowid" varchar NOT NULL, "content" text NOT NULL, "sourceDocuments" text, "createdDate" datetime NOT NULL DEFAULT (datetime('now')), "chatType" VARCHAR NOT NULL DEFAULT 'INTERNAL', "chatId" VARCHAR NOT NULL, "memoryType" VARCHAR, "sessionId" VARCHAR);`
+        )
+        await queryRunner.query(
+            `INSERT INTO "temp_chat_message" ("id", "role", "chatflowid", "content", "sourceDocuments", "createdDate", "chatId") SELECT "id", "role", "chatflowid", "content", "sourceDocuments", "createdDate", "chatId" FROM "chat_message";`
+        )
+        await queryRunner.query(`DROP TABLE "chat_message";`)
+        await queryRunner.query(`ALTER TABLE "temp_chat_message" RENAME TO "chat_message";`)
+        await queryRunner.query(`CREATE INDEX "IDX_e574527322272fd838f4f0f3d3" ON "chat_message" ("chatflowid") ;`)
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`DROP TABLE IF EXISTS "temp_chat_message";`)
+        await queryRunner.query(`ALTER TABLE "chat_message" DROP COLUMN "chatType";`)
+        await queryRunner.query(`ALTER TABLE "chat_message" DROP COLUMN "chatId";`)
+        await queryRunner.query(`ALTER TABLE "chat_message" DROP COLUMN "memoryType";`)
+        await queryRunner.query(`ALTER TABLE "chat_message" DROP COLUMN "sessionId";`)
+    }
+}

packages/server/src/database/migrations/sqlite/index.ts

@@ -5,6 +5,7 @@ import { ModifyCredential1693923551694 } from './1693923551694-ModifyCredential'
 import { ModifyTool1693924207475 } from './1693924207475-ModifyTool'
 import { AddApiConfig1694090982460 } from './1694090982460-AddApiConfig'
 import { AddAnalytic1694432361423 } from './1694432361423-AddAnalytic'
+import { AddChatHistory1694657778173 } from './1694657778173-AddChatHistory'
 
 export const sqliteMigrations = [
     Init1693835579790,
@@ -13,5 +14,6 @@ export const sqliteMigrations = [
     ModifyCredential1693923551694,
     ModifyTool1693924207475,
     AddApiConfig1694090982460,
-    AddAnalytic1694432361423
+    AddAnalytic1694432361423,
+    AddChatHistory1694657778173
 ]

packages/server/src/index.ts

@@ -8,7 +8,8 @@ import basicAuth from 'express-basic-auth'
 import { Server } from 'socket.io'
 import logger from './utils/logger'
 import { expressRequestLogger } from './utils/logger'
-
+import { v4 as uuidv4 } from 'uuid'
+import { Between, IsNull, FindOptionsWhere } from 'typeorm'
 import {
     IChatFlow,
     IncomingInput,
@@ -16,7 +17,10 @@ import {
     IReactFlowObject,
     INodeData,
     IDatabaseExport,
-    ICredentialReturnResponse
+    ICredentialReturnResponse,
+    chatType,
+    IChatMessage,
+    IReactFlowEdge
 } from './Interface'
 import {
     getNodeModulesPackagePath,
@@ -40,10 +44,11 @@ import {
     getApiKey,
     transformToCredentialEntity,
     decryptCredentialData,
-    clearSessionMemory,
+    clearAllSessionMemory,
     replaceInputsWithConfig,
     getEncryptionKey,
-    checkMemorySessionId
+    checkMemorySessionId,
+    clearSessionMemoryFromViewMessageDialog
 } from './utils'
 import { cloneDeep, omit } from 'lodash'
 import { getDataSource } from './DataSource'
@@ -395,45 +400,92 @@ export class App {
 
         // Get all chatmessages from chatflowid
         this.app.get('/api/v1/chatmessage/:id', async (req: Request, res: Response) => {
-            const chatmessages = await this.AppDataSource.getRepository(ChatMessage).find({
+            const sortOrder = req.query?.order as string | undefined
-                where: {
+            const chatId = req.query?.chatId as string | undefined
-                    chatflowid: req.params.id
+            const memoryType = req.query?.memoryType as string | undefined
-                },
+            const sessionId = req.query?.sessionId as string | undefined
-                order: {
+            const startDate = req.query?.startDate as string | undefined
-                    createdDate: 'ASC'
+            const endDate = req.query?.endDate as string | undefined
+            let chatTypeFilter = req.query?.chatType as chatType | undefined
+
+            if (chatTypeFilter) {
+                try {
+                    const chatTypeFilterArray = JSON.parse(chatTypeFilter)
+                    if (chatTypeFilterArray.includes(chatType.EXTERNAL) && chatTypeFilterArray.includes(chatType.INTERNAL)) {
+                        chatTypeFilter = undefined
+                    } else if (chatTypeFilterArray.includes(chatType.EXTERNAL)) {
+                        chatTypeFilter = chatType.EXTERNAL
+                    } else if (chatTypeFilterArray.includes(chatType.INTERNAL)) {
+                        chatTypeFilter = chatType.INTERNAL
+                    }
+                } catch (e) {
+                    return res.status(500).send(e)
                 }
-            })
+            }
+
+            const chatmessages = await this.getChatMessage(
+                req.params.id,
+                chatTypeFilter,
+                sortOrder,
+                chatId,
+                memoryType,
+                sessionId,
+                startDate,
+                endDate
+            )
+            return res.json(chatmessages)
+        })
+
+        // Get internal chatmessages from chatflowid
+        this.app.get('/api/v1/internal-chatmessage/:id', async (req: Request, res: Response) => {
+            const chatmessages = await this.getChatMessage(req.params.id, chatType.INTERNAL)
             return res.json(chatmessages)
         })
 
         // Add chatmessages for chatflowid
         this.app.post('/api/v1/chatmessage/:id', async (req: Request, res: Response) => {
             const body = req.body
-            const newChatMessage = new ChatMessage()
+            const results = await this.addChatMessage(body)
-            Object.assign(newChatMessage, body)
-
-            const chatmessage = this.AppDataSource.getRepository(ChatMessage).create(newChatMessage)
-            const results = await this.AppDataSource.getRepository(ChatMessage).save(chatmessage)
-
             return res.json(results)
         })
 
-        // Delete all chatmessages from chatflowid
+        // Delete all chatmessages from chatId
         this.app.delete('/api/v1/chatmessage/:id', async (req: Request, res: Response) => {
+            const chatflowid = req.params.id
             const chatflow = await this.AppDataSource.getRepository(ChatFlow).findOneBy({
-                id: req.params.id
+                id: chatflowid
             })
             if (!chatflow) {
-                res.status(404).send(`Chatflow ${req.params.id} not found`)
+                res.status(404).send(`Chatflow ${chatflowid} not found`)
                 return
             }
+            const chatId = (req.query?.chatId as string) ?? (await getChatId(chatflowid))
+            const memoryType = req.query?.memoryType as string | undefined
+            const sessionId = req.query?.sessionId as string | undefined
+            const chatType = req.query?.chatType as string | undefined
+            const isClearFromViewMessageDialog = req.query?.isClearFromViewMessageDialog as string | undefined
+
             const flowData = chatflow.flowData
             const parsedFlowData: IReactFlowObject = JSON.parse(flowData)
             const nodes = parsedFlowData.nodes
-            let chatId = await getChatId(chatflow.id)
+
-            if (!chatId) chatId = chatflow.id
+            if (isClearFromViewMessageDialog)
-            clearSessionMemory(nodes, this.nodesPool.componentNodes, chatId, this.AppDataSource, req.query.sessionId as string)
+                clearSessionMemoryFromViewMessageDialog(
-            const results = await this.AppDataSource.getRepository(ChatMessage).delete({ chatflowid: req.params.id })
+                    nodes,
+                    this.nodesPool.componentNodes,
+                    chatId,
+                    this.AppDataSource,
+                    sessionId,
+                    memoryType
+                )
+            else clearAllSessionMemory(nodes, this.nodesPool.componentNodes, chatId, this.AppDataSource, sessionId)
+
+            const deleteOptions: FindOptionsWhere<ChatMessage> = { chatflowid, chatId }
+            if (memoryType) deleteOptions.memoryType = memoryType
+            if (sessionId) deleteOptions.sessionId = sessionId
+            if (chatType) deleteOptions.chatType = chatType
+
+            const results = await this.AppDataSource.getRepository(ChatMessage).delete(deleteOptions)
             return res.json(results)
         })
 
@@ -809,18 +861,95 @@ export class App {
      * @param {Response} res
      * @param {ChatFlow} chatflow
      */
-    async validateKey(req: Request, res: Response, chatflow: ChatFlow) {
+    async validateKey(req: Request, chatflow: ChatFlow) {
         const chatFlowApiKeyId = chatflow.apikeyid
-        const authorizationHeader = (req.headers['Authorization'] as string) ?? (req.headers['authorization'] as string) ?? ''
+        if (!chatFlowApiKeyId) return true
 
-        if (chatFlowApiKeyId && !authorizationHeader) return res.status(401).send(`Unauthorized`)
+        const authorizationHeader = (req.headers['Authorization'] as string) ?? (req.headers['authorization'] as string) ?? ''
+        if (chatFlowApiKeyId && !authorizationHeader) return false
 
         const suppliedKey = authorizationHeader.split(`Bearer `).pop()
-        if (chatFlowApiKeyId && suppliedKey) {
+        if (suppliedKey) {
             const keys = await getAPIKeys()
             const apiSecret = keys.find((key) => key.id === chatFlowApiKeyId)?.apiSecret
-            if (!compareKeys(apiSecret, suppliedKey)) return res.status(401).send(`Unauthorized`)
+            if (!compareKeys(apiSecret, suppliedKey)) return false
+            return true
         }
+        return false
+    }
+
+    /**
+     * Method that get chat messages.
+     * @param {string} chatflowid
+     * @param {chatType} chatType
+     * @param {string} sortOrder
+     * @param {string} chatId
+     * @param {string} memoryType
+     * @param {string} sessionId
+     * @param {string} startDate
+     * @param {string} endDate
+     */
+    async getChatMessage(
+        chatflowid: string,
+        chatType: chatType | undefined,
+        sortOrder: string = 'ASC',
+        chatId?: string,
+        memoryType?: string,
+        sessionId?: string,
+        startDate?: string,
+        endDate?: string
+    ): Promise<ChatMessage[]> {
+        let fromDate
+        if (startDate) fromDate = new Date(startDate)
+
+        let toDate
+        if (endDate) toDate = new Date(endDate)
+
+        return await this.AppDataSource.getRepository(ChatMessage).find({
+            where: {
+                chatflowid,
+                chatType,
+                chatId,
+                memoryType: memoryType ?? (chatId ? IsNull() : undefined),
+                sessionId: sessionId ?? (chatId ? IsNull() : undefined),
+                createdDate: toDate && fromDate ? Between(fromDate, toDate) : undefined
+            },
+            order: {
+                createdDate: sortOrder === 'DESC' ? 'DESC' : 'ASC'
+            }
+        })
+    }
+
+    /**
+     * Method that add chat messages.
+     * @param {Partial<IChatMessage>} chatMessage
+     */
+    async addChatMessage(chatMessage: Partial<IChatMessage>): Promise<ChatMessage> {
+        const newChatMessage = new ChatMessage()
+        Object.assign(newChatMessage, chatMessage)
+
+        const chatmessage = this.AppDataSource.getRepository(ChatMessage).create(newChatMessage)
+        return await this.AppDataSource.getRepository(ChatMessage).save(chatmessage)
+    }
+
+    /**
+     * Method that find memory label that is connected within chatflow
+     * In a chatflow, there should only be 1 memory node
+     * @param {IReactFlowNode[]} nodes
+     * @param {IReactFlowEdge[]} edges
+     * @returns {string | undefined}
+     */
+    findMemoryLabel(nodes: IReactFlowNode[], edges: IReactFlowEdge[]): string | undefined {
+        const memoryNodes = nodes.filter((node) => node.data.category === 'Memory')
+        const memoryNodeIds = memoryNodes.map((mem) => mem.data.id)
+
+        for (const edge of edges) {
+            if (memoryNodeIds.includes(edge.source)) {
+                const memoryNode = nodes.find((node) => node.data.id === edge.source)
+                return memoryNode ? memoryNode.data.label : undefined
+            }
+        }
+        return undefined
     }
 
     /**
@@ -830,7 +959,7 @@ export class App {
      * @param {Server} socketIO
      * @param {boolean} isInternal
      */
-    async processPrediction(req: Request, res: Response, socketIO?: Server, isInternal = false) {
+    async processPrediction(req: Request, res: Response, socketIO?: Server, isInternal: boolean = false) {
         try {
             const chatflowid = req.params.id
             let incomingInput: IncomingInput = req.body
@@ -842,11 +971,12 @@ export class App {
             })
             if (!chatflow) return res.status(404).send(`Chatflow ${chatflowid} not found`)
 
-            let chatId = await getChatId(chatflow.id)
+            const chatId = incomingInput.chatId ?? incomingInput.overrideConfig?.sessionId ?? uuidv4()
-            if (!chatId) chatId = chatflowid
+            const userMessageDateTime = new Date()
 
             if (!isInternal) {
-                await this.validateKey(req, res, chatflow)
+                const isKeyValidated = await this.validateKey(req, chatflow)
+                if (!isKeyValidated) return res.status(401).send('Unauthorized')
             }
 
             let isStreamValid = false
@@ -978,9 +1108,12 @@ export class App {
 
             logger.debug(`[server]: Running ${nodeToExecuteData.label} (${nodeToExecuteData.id})`)
 
-            if (nodeToExecuteData.instance) checkMemorySessionId(nodeToExecuteData.instance, chatId)
+            let sessionId = undefined
+            if (nodeToExecuteData.instance) sessionId = checkMemorySessionId(nodeToExecuteData.instance, chatId)
 
-            const result = isStreamValid
+            const memoryType = this.findMemoryLabel(nodes, edges)
+
+            let result = isStreamValid
                 ? await nodeInstance.run(nodeToExecuteData, incomingInput.question, {
                       chatHistory: incomingInput.history,
                       socketIO,
@@ -998,7 +1131,42 @@ export class App {
                       analytic: chatflow.analytic
                   })
 
+            result = typeof result === 'string' ? { text: result } : result
+
+            const userMessage: Omit<IChatMessage, 'id'> = {
+                role: 'userMessage',
+                content: incomingInput.question,
+                chatflowid,
+                chatType: isInternal ? chatType.INTERNAL : chatType.EXTERNAL,
+                chatId,
+                memoryType,
+                sessionId,
+                createdDate: userMessageDateTime
+            }
+            await this.addChatMessage(userMessage)
+
+            let resultText = ''
+            if (result.text) resultText = result.text
+            else if (result.json) resultText = '```json\n' + JSON.stringify(result.json, null, 2)
+            else resultText = JSON.stringify(result, null, 2)
+
+            const apiMessage: Omit<IChatMessage, 'id' | 'createdDate'> = {
+                role: 'apiMessage',
+                content: resultText,
+                chatflowid,
+                chatType: isInternal ? chatType.INTERNAL : chatType.EXTERNAL,
+                chatId,
+                memoryType,
+                sessionId
+            }
+            if (result?.sourceDocuments) apiMessage.sourceDocuments = JSON.stringify(result.sourceDocuments)
+            await this.addChatMessage(apiMessage)
+
             logger.debug(`[server]: Finished running ${nodeToExecuteData.label} (${nodeToExecuteData.id})`)
+
+            // Only return ChatId when its Internal OR incoming input has ChatId, to avoid confusion when calling API
+            if (incomingInput.chatId || isInternal) result.chatId = chatId
+
             return res.json(result)
         } catch (e: any) {
             logger.error('[server]: Error:', e)
@@ -1021,7 +1189,7 @@ export class App {
  * @param {string} chatflowid
  * @returns {string}
  */
-export async function getChatId(chatflowid: string) {
+export async function getChatId(chatflowid: string): Promise<string> {
     // first chatmessage id as the unique chat id
     const firstChatMessage = await getDataSource()
         .getRepository(ChatMessage)

packages/server/src/utils/index.ts

@@ -298,14 +298,14 @@ export const buildLangchain = async (
 }
 
 /**
- * Clear memory
+ * Clear all session memories on the canvas
  * @param {IReactFlowNode[]} reactFlowNodes
  * @param {IComponentNodes} componentNodes
  * @param {string} chatId
  * @param {DataSource} appDataSource
  * @param {string} sessionId
  */
-export const clearSessionMemory = async (
+export const clearAllSessionMemory = async (
     reactFlowNodes: IReactFlowNode[],
     componentNodes: IComponentNodes,
     chatId: string,
@@ -317,9 +317,46 @@ export const clearSessionMemory = async (
         const nodeInstanceFilePath = componentNodes[node.data.name].filePath as string
         const nodeModule = await import(nodeInstanceFilePath)
         const newNodeInstance = new nodeModule.nodeClass()
+
         if (sessionId && node.data.inputs) node.data.inputs.sessionId = sessionId
-        if (newNodeInstance.clearSessionMemory)
+
+        if (newNodeInstance.clearSessionMemory) {
             await newNodeInstance?.clearSessionMemory(node.data, { chatId, appDataSource, databaseEntities, logger })
+        }
+    }
+}
+
+/**
+ * Clear specific session memory from View Message Dialog UI
+ * @param {IReactFlowNode[]} reactFlowNodes
+ * @param {IComponentNodes} componentNodes
+ * @param {string} chatId
+ * @param {DataSource} appDataSource
+ * @param {string} sessionId
+ * @param {string} memoryType
+ */
+export const clearSessionMemoryFromViewMessageDialog = async (
+    reactFlowNodes: IReactFlowNode[],
+    componentNodes: IComponentNodes,
+    chatId: string,
+    appDataSource: DataSource,
+    sessionId?: string,
+    memoryType?: string
+) => {
+    if (!sessionId) return
+    for (const node of reactFlowNodes) {
+        if (node.data.category !== 'Memory') continue
+        if (node.data.label !== memoryType) continue
+        const nodeInstanceFilePath = componentNodes[node.data.name].filePath as string
+        const nodeModule = await import(nodeInstanceFilePath)
+        const newNodeInstance = new nodeModule.nodeClass()
+
+        if (sessionId && node.data.inputs) node.data.inputs.sessionId = sessionId
+
+        if (newNodeInstance.clearSessionMemory) {
+            await newNodeInstance?.clearSessionMemory(node.data, { chatId, appDataSource, databaseEntities, logger })
+            return
+        }
     }
 }
 
@@ -937,8 +974,10 @@ export const redactCredentialWithPasswordType = (
  * @param {any} instance
  * @param {string} chatId
  */
-export const checkMemorySessionId = (instance: any, chatId: string) => {
+export const checkMemorySessionId = (instance: any, chatId: string): string => {
     if (instance.memory && instance.memory.isSessionIdUsingChatMessageId && chatId) {
         instance.memory.sessionId = chatId
+        instance.memory.chatHistory.sessionId = chatId
     }
+    return instance.memory ? instance.memory.sessionId ?? instance.memory.chatHistory.sessionId : undefined
 }

packages/ui/package.json

@@ -31,7 +31,7 @@
         "react": "^18.2.0",
         "react-code-blocks": "^0.0.9-0",
         "react-color": "^2.19.3",
-        "react-datepicker": "^4.8.0",
+        "react-datepicker": "^4.21.0",
         "react-device-detect": "^1.17.0",
         "react-dom": "^18.2.0",
         "react-markdown": "^8.0.6",

packages/ui/src/api/chatmessage.js

@@ -1,13 +1,13 @@
 import client from './client'
 
-const getChatmessageFromChatflow = (id) => client.get(`/chatmessage/${id}`)
+const getInternalChatmessageFromChatflow = (id) => client.get(`/internal-chatmessage/${id}`)
-
+const getAllChatmessageFromChatflow = (id, params = {}) => client.get(`/chatmessage/${id}`, { params: { order: 'DESC', ...params } })
-const createNewChatmessage = (id, body) => client.post(`/chatmessage/${id}`, body)
+const getChatmessageFromPK = (id, params = {}) => client.get(`/chatmessage/${id}`, { params: { order: 'ASC', ...params } })
-
+const deleteChatmessage = (id, params = {}) => client.delete(`/chatmessage/${id}`, { params: { ...params } })
-const deleteChatmessage = (id) => client.delete(`/chatmessage/${id}`)
 
 export default {
-    getChatmessageFromChatflow,
+    getInternalChatmessageFromChatflow,
-    createNewChatmessage,
+    getAllChatmessageFromChatflow,
+    getChatmessageFromPK,
     deleteChatmessage
 }

packages/ui/src/layout/MainLayout/Header/ProfileSection/index.js

@@ -71,7 +71,7 @@ const ProfileSection = ({ username, handleLogout }) => {
         try {
             const response = await databaseApi.getExportDatabase()
             const exportItems = response.data
-            let dataStr = JSON.stringify(exportItems)
+            let dataStr = JSON.stringify(exportItems, null, 2)
             let dataUri = 'data:application/json;charset=utf-8,' + encodeURIComponent(dataStr)
 
             let exportFileDefaultName = `DB.json`

packages/ui/src/menu-items/settings.js

@@ -1,8 +1,8 @@
 // assets
-import { IconTrash, IconFileUpload, IconFileExport, IconCopy, IconSearch } from '@tabler/icons'
+import { IconTrash, IconFileUpload, IconFileExport, IconCopy, IconSearch, IconMessage } from '@tabler/icons'
 
 // constant
-const icons = { IconTrash, IconFileUpload, IconFileExport, IconCopy, IconSearch }
+const icons = { IconTrash, IconFileUpload, IconFileExport, IconCopy, IconSearch, IconMessage }
 
 // ==============================|| SETTINGS MENU ITEMS ||============================== //
 
@@ -11,6 +11,13 @@ const settings = {
     title: '',
     type: 'group',
     children: [
+        {
+            id: 'viewMessages',
+            title: 'View Messages',
+            type: 'item',
+            url: '',
+            icon: icons.IconMessage
+        },
         {
             id: 'duplicateChatflow',
             title: 'Duplicate Chatflow',

packages/ui/src/themes/palette.js

@@ -80,6 +80,9 @@ export default function themePalette(theme) {
         asyncSelect: {
             main: theme.customization.isDarkMode ? theme.colors?.darkPrimary800 : theme.colors?.grey50
         },
+        timeMessage: {
+            main: theme.customization.isDarkMode ? theme.colors?.darkLevel2 : theme.colors?.grey200
+        },
         canvasHeader: {
             deployLight: theme.colors?.primaryLight,
             deployDark: theme.colors?.primaryDark,

packages/ui/src/ui-component/dialog/ViewMessagesDialog.js

@@ -0,0 +1,695 @@
+import { createPortal } from 'react-dom'
+import { useDispatch, useSelector } from 'react-redux'
+import { useState, useEffect, forwardRef } from 'react'
+import PropTypes from 'prop-types'
+import moment from 'moment'
+import rehypeMathjax from 'rehype-mathjax'
+import remarkGfm from 'remark-gfm'
+import remarkMath from 'remark-math'
+
+// material-ui
+import {
+    Button,
+    Tooltip,
+    ListItemButton,
+    Box,
+    Stack,
+    Dialog,
+    DialogContent,
+    DialogTitle,
+    ListItem,
+    ListItemText,
+    Chip
+} from '@mui/material'
+import { useTheme } from '@mui/material/styles'
+import DatePicker from 'react-datepicker'
+
+import robotPNG from 'assets/images/robot.png'
+import userPNG from 'assets/images/account.png'
+import msgEmptySVG from 'assets/images/message_empty.svg'
+import { IconFileExport, IconEraser, IconX } from '@tabler/icons'
+
+// Project import
+import { MemoizedReactMarkdown } from 'ui-component/markdown/MemoizedReactMarkdown'
+import { CodeBlock } from 'ui-component/markdown/CodeBlock'
+import SourceDocDialog from 'ui-component/dialog/SourceDocDialog'
+import { MultiDropdown } from 'ui-component/dropdown/MultiDropdown'
+import { StyledButton } from 'ui-component/button/StyledButton'
+
+// store
+import { HIDE_CANVAS_DIALOG, SHOW_CANVAS_DIALOG } from 'store/actions'
+
+// API
+import chatmessageApi from 'api/chatmessage'
+import useApi from 'hooks/useApi'
+import useConfirm from 'hooks/useConfirm'
+
+// Utils
+import { isValidURL, removeDuplicateURL } from 'utils/genericHelper'
+import useNotifier from 'utils/useNotifier'
+
+import { enqueueSnackbar as enqueueSnackbarAction, closeSnackbar as closeSnackbarAction } from 'store/actions'
+
+import 'views/chatmessage/ChatMessage.css'
+import 'react-datepicker/dist/react-datepicker.css'
+
+const DatePickerCustomInput = forwardRef(function DatePickerCustomInput({ value, onClick }, ref) {
+    return (
+        <ListItemButton style={{ borderRadius: 15, border: '1px solid #e0e0e0' }} onClick={onClick} ref={ref}>
+            {value}
+        </ListItemButton>
+    )
+})
+
+DatePickerCustomInput.propTypes = {
+    value: PropTypes.string,
+    onClick: PropTypes.func
+}
+
+const ViewMessagesDialog = ({ show, dialogProps, onCancel }) => {
+    const portalElement = document.getElementById('portal')
+    const dispatch = useDispatch()
+    const theme = useTheme()
+    const customization = useSelector((state) => state.customization)
+    const { confirm } = useConfirm()
+
+    useNotifier()
+    const enqueueSnackbar = (...args) => dispatch(enqueueSnackbarAction(...args))
+    const closeSnackbar = (...args) => dispatch(closeSnackbarAction(...args))
+
+    const [chatlogs, setChatLogs] = useState([])
+    const [allChatlogs, setAllChatLogs] = useState([])
+    const [chatMessages, setChatMessages] = useState([])
+    const [selectedMessageIndex, setSelectedMessageIndex] = useState(0)
+    const [sourceDialogOpen, setSourceDialogOpen] = useState(false)
+    const [sourceDialogProps, setSourceDialogProps] = useState({})
+    const [chatTypeFilter, setChatTypeFilter] = useState([])
+    const [startDate, setStartDate] = useState(new Date().setMonth(new Date().getMonth() - 1))
+    const [endDate, setEndDate] = useState(new Date())
+
+    const getChatmessageApi = useApi(chatmessageApi.getAllChatmessageFromChatflow)
+    const getChatmessageFromPKApi = useApi(chatmessageApi.getChatmessageFromPK)
+
+    const onStartDateSelected = (date) => {
+        setStartDate(date)
+        getChatmessageApi.request(dialogProps.chatflow.id, {
+            startDate: date,
+            endDate: endDate,
+            chatType: chatTypeFilter.length ? chatTypeFilter : undefined
+        })
+    }
+
+    const onEndDateSelected = (date) => {
+        setEndDate(date)
+        getChatmessageApi.request(dialogProps.chatflow.id, {
+            endDate: date,
+            startDate: startDate,
+            chatType: chatTypeFilter.length ? chatTypeFilter : undefined
+        })
+    }
+
+    const onChatTypeSelected = (chatTypes) => {
+        setChatTypeFilter(chatTypes)
+        getChatmessageApi.request(dialogProps.chatflow.id, {
+            chatType: chatTypes.length ? chatTypes : undefined,
+            startDate: startDate,
+            endDate: endDate
+        })
+    }
+
+    const exportMessages = () => {
+        const obj = {}
+        for (let i = 0; i < allChatlogs.length; i += 1) {
+            const chatmsg = allChatlogs[i]
+            const chatPK = getChatPK(chatmsg)
+            const msg = {
+                content: chatmsg.content,
+                role: chatmsg.role === 'apiMessage' ? 'bot' : 'user',
+                time: chatmsg.createdDate
+            }
+            if (chatmsg.sourceDocuments) msg.sourceDocuments = JSON.parse(chatmsg.sourceDocuments)
+
+            if (!Object.prototype.hasOwnProperty.call(obj, chatPK)) {
+                obj[chatPK] = {
+                    id: chatmsg.chatId,
+                    source: chatmsg.chatType === 'INTERNAL' ? 'UI' : 'API/Embed',
+                    sessionId: chatmsg.sessionId ?? null,
+                    memoryType: chatmsg.memoryType ?? null,
+                    messages: [msg]
+                }
+            } else if (Object.prototype.hasOwnProperty.call(obj, chatPK)) {
+                obj[chatPK].messages = [...obj[chatPK].messages, msg]
+            }
+        }
+
+        const exportMessages = []
+        for (const key in obj) {
+            exportMessages.push({
+                ...obj[key]
+            })
+        }
+
+        for (let i = 0; i < exportMessages.length; i += 1) {
+            exportMessages[i].messages = exportMessages[i].messages.reverse()
+        }
+
+        const dataStr = JSON.stringify(exportMessages, null, 2)
+        const dataUri = 'data:application/json;charset=utf-8,' + encodeURIComponent(dataStr)
+
+        const exportFileDefaultName = `${dialogProps.chatflow.id}-Message.json`
+
+        let linkElement = document.createElement('a')
+        linkElement.setAttribute('href', dataUri)
+        linkElement.setAttribute('download', exportFileDefaultName)
+        linkElement.click()
+    }
+
+    const clearChat = async (chatmsg) => {
+        const description =
+            chatmsg.sessionId && chatmsg.memoryType
+                ? `Are you sure you want to clear session id: ${chatmsg.sessionId} from ${chatmsg.memoryType}?`
+                : `Are you sure you want to clear messages?`
+        const confirmPayload = {
+            title: `Clear Session`,
+            description,
+            confirmButtonName: 'Clear',
+            cancelButtonName: 'Cancel'
+        }
+        const isConfirmed = await confirm(confirmPayload)
+
+        const chatflowid = dialogProps.chatflow.id
+        if (isConfirmed) {
+            try {
+                const obj = { chatflowid, isClearFromViewMessageDialog: true }
+                if (chatmsg.chatId) obj.chatId = chatmsg.chatId
+                if (chatmsg.chatType) obj.chatType = chatmsg.chatType
+                if (chatmsg.memoryType) obj.memoryType = chatmsg.memoryType
+                if (chatmsg.sessionId) obj.sessionId = chatmsg.sessionId
+
+                await chatmessageApi.deleteChatmessage(chatflowid, obj)
+                const description =
+                    chatmsg.sessionId && chatmsg.memoryType
+                        ? `Succesfully cleared session id: ${chatmsg.sessionId} from ${chatmsg.memoryType}`
+                        : `Succesfully cleared messages`
+                enqueueSnackbar({
+                    message: description,
+                    options: {
+                        key: new Date().getTime() + Math.random(),
+                        variant: 'success',
+                        action: (key) => (
+                            <Button style={{ color: 'white' }} onClick={() => closeSnackbar(key)}>
+                                <IconX />
+                            </Button>
+                        )
+                    }
+                })
+                getChatmessageApi.request(chatflowid)
+            } catch (error) {
+                const errorData = error.response.data || `${error.response.status}: ${error.response.statusText}`
+                enqueueSnackbar({
+                    message: errorData,
+                    options: {
+                        key: new Date().getTime() + Math.random(),
+                        variant: 'error',
+                        persist: true,
+                        action: (key) => (
+                            <Button style={{ color: 'white' }} onClick={() => closeSnackbar(key)}>
+                                <IconX />
+                            </Button>
+                        )
+                    }
+                })
+            }
+        }
+    }
+
+    const getChatMessages = (chatmessages) => {
+        let prevDate = ''
+        const loadedMessages = []
+        for (let i = 0; i < chatmessages.length; i += 1) {
+            const chatmsg = chatmessages[i]
+            if (!prevDate) {
+                prevDate = chatmsg.createdDate.split('T')[0]
+                loadedMessages.push({
+                    message: chatmsg.createdDate,
+                    type: 'timeMessage'
+                })
+            } else {
+                const currentDate = chatmsg.createdDate.split('T')[0]
+                if (currentDate !== prevDate) {
+                    prevDate = currentDate
+                    loadedMessages.push({
+                        message: chatmsg.createdDate,
+                        type: 'timeMessage'
+                    })
+                }
+            }
+            const obj = {
+                ...chatmsg,
+                message: chatmsg.content,
+                type: chatmsg.role
+            }
+            if (chatmsg.sourceDocuments) obj.sourceDocuments = JSON.parse(chatmsg.sourceDocuments)
+            loadedMessages.push(obj)
+        }
+        setChatMessages(loadedMessages)
+    }
+
+    const getChatPK = (chatmsg) => {
+        const chatId = chatmsg.chatId
+        const memoryType = chatmsg.memoryType ?? 'null'
+        const sessionId = chatmsg.sessionId ?? 'null'
+        return `${chatId}_${memoryType}_${sessionId}`
+    }
+
+    const transformChatPKToParams = (chatPK) => {
+        const chatId = chatPK.split('_')[0]
+        const memoryType = chatPK.split('_')[1]
+        const sessionId = chatPK.split('_')[2]
+
+        const params = { chatId }
+        if (memoryType !== 'null') params.memoryType = memoryType
+        if (sessionId !== 'null') params.sessionId = sessionId
+
+        return params
+    }
+
+    const processChatLogs = (allChatMessages) => {
+        const seen = {}
+        const filteredChatLogs = []
+        for (let i = 0; i < allChatMessages.length; i += 1) {
+            const PK = getChatPK(allChatMessages[i])
+
+            const item = allChatMessages[i]
+            if (!Object.prototype.hasOwnProperty.call(seen, PK)) {
+                seen[PK] = {
+                    counter: 1,
+                    item: allChatMessages[i]
+                }
+            } else if (Object.prototype.hasOwnProperty.call(seen, PK) && seen[PK].counter === 1) {
+                seen[PK] = {
+                    counter: 2,
+                    item: {
+                        ...seen[PK].item,
+                        apiContent:
+                            seen[PK].item.role === 'apiMessage' ? `Bot: ${seen[PK].item.content}` : `User: ${seen[PK].item.content}`,
+                        userContent: item.role === 'apiMessage' ? `Bot: ${item.content}` : `User: ${item.content}`
+                    }
+                }
+                filteredChatLogs.push(seen[PK].item)
+            }
+        }
+        setChatLogs(filteredChatLogs)
+        if (filteredChatLogs.length) return getChatPK(filteredChatLogs[0])
+        return undefined
+    }
+
+    const handleItemClick = (idx, chatmsg) => {
+        setSelectedMessageIndex(idx)
+        getChatmessageFromPKApi.request(dialogProps.chatflow.id, transformChatPKToParams(getChatPK(chatmsg)))
+    }
+
+    const onURLClick = (data) => {
+        window.open(data, '_blank')
+    }
+
+    const onSourceDialogClick = (data) => {
+        setSourceDialogProps({ data })
+        setSourceDialogOpen(true)
+    }
+
+    useEffect(() => {
+        if (getChatmessageFromPKApi.data) {
+            getChatMessages(getChatmessageFromPKApi.data)
+        }
+
+        // eslint-disable-next-line react-hooks/exhaustive-deps
+    }, [getChatmessageFromPKApi.data])
+
+    useEffect(() => {
+        if (getChatmessageApi.data) {
+            setAllChatLogs(getChatmessageApi.data)
+            const chatPK = processChatLogs(getChatmessageApi.data)
+            setSelectedMessageIndex(0)
+            if (chatPK) getChatmessageFromPKApi.request(dialogProps.chatflow.id, transformChatPKToParams(chatPK))
+        }
+
+        // eslint-disable-next-line react-hooks/exhaustive-deps
+    }, [getChatmessageApi.data])
+
+    useEffect(() => {
+        if (dialogProps.chatflow) {
+            getChatmessageApi.request(dialogProps.chatflow.id)
+        }
+
+        return () => {
+            setChatLogs([])
+            setAllChatLogs([])
+            setChatMessages([])
+            setChatTypeFilter([])
+            setSelectedMessageIndex(0)
+            setStartDate(new Date().setMonth(new Date().getMonth() - 1))
+            setEndDate(new Date())
+        }
+
+        // eslint-disable-next-line react-hooks/exhaustive-deps
+    }, [dialogProps])
+
+    useEffect(() => {
+        if (show) dispatch({ type: SHOW_CANVAS_DIALOG })
+        else dispatch({ type: HIDE_CANVAS_DIALOG })
+        return () => dispatch({ type: HIDE_CANVAS_DIALOG })
+    }, [show, dispatch])
+
+    const component = show ? (
+        <Dialog
+            onClose={onCancel}
+            open={show}
+            fullWidth
+            maxWidth={chatlogs && chatlogs.length == 0 ? 'md' : 'lg'}
+            aria-labelledby='alert-dialog-title'
+            aria-describedby='alert-dialog-description'
+        >
+            <DialogTitle sx={{ fontSize: '1rem' }} id='alert-dialog-title'>
+                <div style={{ display: 'flex', flexDirection: 'row' }}>
+                    {dialogProps.title}
+                    <div style={{ flex: 1 }} />
+                    <Button variant='outlined' onClick={() => exportMessages()} startIcon={<IconFileExport />}>
+                        Export
+                    </Button>
+                </div>
+            </DialogTitle>
+            <DialogContent>
+                <>
+                    <div style={{ display: 'flex', flexDirection: 'row', alignItems: 'center', width: '100%', marginBottom: 10 }}>
+                        <div style={{ marginRight: 10 }}>
+                            <b style={{ marginRight: 10 }}>From Date</b>
+                            <DatePicker
+                                selected={startDate}
+                                onChange={(date) => onStartDateSelected(date)}
+                                selectsStart
+                                startDate={startDate}
+                                endDate={endDate}
+                                customInput={<DatePickerCustomInput />}
+                            />
+                        </div>
+                        <div style={{ marginRight: 10 }}>
+                            <b style={{ marginRight: 10 }}>To Date</b>
+                            <DatePicker
+                                selected={endDate}
+                                onChange={(date) => onEndDateSelected(date)}
+                                selectsEnd
+                                startDate={startDate}
+                                endDate={endDate}
+                                minDate={startDate}
+                                maxDate={new Date()}
+                                customInput={<DatePickerCustomInput />}
+                            />
+                        </div>
+                        <div style={{ display: 'flex', flexDirection: 'row', alignItems: 'center', minWidth: '200px', marginRight: 10 }}>
+                            <b style={{ marginRight: 10 }}>Source</b>
+                            <MultiDropdown
+                                key={JSON.stringify(chatTypeFilter)}
+                                name='chatType'
+                                options={[
+                                    {
+                                        label: 'UI',
+                                        name: 'INTERNAL'
+                                    },
+                                    {
+                                        label: 'API/Embed',
+                                        name: 'EXTERNAL'
+                                    }
+                                ]}
+                                onSelect={(newValue) => onChatTypeSelected(newValue)}
+                                value={chatTypeFilter}
+                                formControlSx={{ mt: 0 }}
+                            />
+                        </div>
+                        <div style={{ flex: 1 }}></div>
+                    </div>
+                    <div style={{ display: 'flex', flexDirection: 'row' }}>
+                        {chatlogs && chatlogs.length == 0 && (
+                            <Stack sx={{ alignItems: 'center', justifyContent: 'center', width: '100%' }} flexDirection='column'>
+                                <Box sx={{ p: 5, height: 'auto' }}>
+                                    <img
+                                        style={{ objectFit: 'cover', height: '20vh', width: 'auto' }}
+                                        src={msgEmptySVG}
+                                        alt='msgEmptySVG'
+                                    />
+                                </Box>
+                                <div>No Messages</div>
+                            </Stack>
+                        )}
+                        {chatlogs && chatlogs.length > 0 && (
+                            <div style={{ flexBasis: '40%' }}>
+                                <Box
+                                    sx={{
+                                        overflowY: 'auto',
+                                        display: 'flex',
+                                        flexGrow: 1,
+                                        flexDirection: 'column',
+                                        maxHeight: 'calc(100vh - 260px)'
+                                    }}
+                                >
+                                    {chatlogs.map((chatmsg, index) => (
+                                        <ListItemButton
+                                            key={index}
+                                            sx={{
+                                                p: 0,
+                                                borderRadius: `${customization.borderRadius}px`,
+                                                boxShadow: '0 2px 14px 0 rgb(32 40 45 / 8%)',
+                                                mt: 1,
+                                                ml: 1,
+                                                mr: 1,
+                                                mb: index === chatlogs.length - 1 ? 1 : 0
+                                            }}
+                                            selected={selectedMessageIndex === index}
+                                            onClick={() => handleItemClick(index, chatmsg)}
+                                        >
+                                            <ListItem alignItems='center'>
+                                                <ListItemText
+                                                    primary={
+                                                        <div style={{ display: 'flex', flexDirection: 'column', marginBottom: 10 }}>
+                                                            <span>{chatmsg?.userContent}</span>
+                                                            <div
+                                                                style={{
+                                                                    maxHeight: '100px',
+                                                                    maxWidth: '400px',
+                                                                    whiteSpace: 'nowrap',
+                                                                    overflow: 'hidden',
+                                                                    textOverflow: 'ellipsis'
+                                                                }}
+                                                            >
+                                                                {chatmsg?.apiContent}
+                                                            </div>
+                                                        </div>
+                                                    }
+                                                    secondary={moment(chatmsg.createdDate).format('MMMM Do YYYY, h:mm:ss a')}
+                                                />
+                                            </ListItem>
+                                        </ListItemButton>
+                                    ))}
+                                </Box>
+                            </div>
+                        )}
+                        {chatlogs && chatlogs.length > 0 && (
+                            <div style={{ flexBasis: '60%', paddingRight: '30px' }}>
+                                {chatMessages && chatMessages.length > 1 && (
+                                    <div style={{ display: 'flex', flexDirection: 'row' }}>
+                                        <div style={{ flex: 1, marginLeft: '20px', marginBottom: '15px', marginTop: '10px' }}>
+                                            {chatMessages[1].sessionId && (
+                                                <div>
+                                                    Session Id:&nbsp;<b>{chatMessages[1].sessionId}</b>
+                                                </div>
+                                            )}
+                                            {chatMessages[1].chatType && (
+                                                <div>
+                                                    Source:&nbsp;<b>{chatMessages[1].chatType === 'INTERNAL' ? 'UI' : 'API/Embed'}</b>
+                                                </div>
+                                            )}
+                                            {chatMessages[1].memoryType && (
+                                                <div>
+                                                    Memory:&nbsp;<b>{chatMessages[1].memoryType}</b>
+                                                </div>
+                                            )}
+                                        </div>
+                                        <div
+                                            style={{
+                                                display: 'flex',
+                                                flexDirection: 'column',
+                                                alignContent: 'center',
+                                                alignItems: 'end'
+                                            }}
+                                        >
+                                            <StyledButton
+                                                sx={{ height: 'max-content', width: 'max-content' }}
+                                                variant='outlined'
+                                                color='error'
+                                                title='Clear Message'
+                                                onClick={() => clearChat(chatMessages[1])}
+                                                startIcon={<IconEraser />}
+                                            >
+                                                Clear
+                                            </StyledButton>
+                                            {chatMessages[1].sessionId && (
+                                                <Tooltip
+                                                    title={
+                                                        'At your left 👈 you will see the Memory node that was used in this conversation. You need to have the matching Memory node with same parameters in the canvas, in order to delete the session conversations stored on the Memory node'
+                                                    }
+                                                    placement='bottom'
+                                                >
+                                                    <h5 style={{ cursor: 'pointer', color: theme.palette.primary.main }}>
+                                                        Why my session is not deleted?
+                                                    </h5>
+                                                </Tooltip>
+                                            )}
+                                        </div>
+                                    </div>
+                                )}
+                                <div
+                                    style={{
+                                        marginLeft: '20px',
+                                        border: '1px solid #e0e0e0',
+                                        borderRadius: `${customization.borderRadius}px`
+                                    }}
+                                    className='cloud-message'
+                                >
+                                    <div style={{ width: '100%', height: '100%' }}>
+                                        {chatMessages &&
+                                            chatMessages.map((message, index) => {
+                                                if (message.type === 'apiMessage' || message.type === 'userMessage') {
+                                                    return (
+                                                        <Box
+                                                            sx={{
+                                                                background:
+                                                                    message.type === 'apiMessage' ? theme.palette.asyncSelect.main : '',
+                                                                pl: 1,
+                                                                pr: 1
+                                                            }}
+                                                            key={index}
+                                                            style={{ display: 'flex', justifyContent: 'center', alignContent: 'center' }}
+                                                        >
+                                                            {/* Display the correct icon depending on the message type */}
+                                                            {message.type === 'apiMessage' ? (
+                                                                <img
+                                                                    style={{ marginLeft: '10px' }}
+                                                                    src={robotPNG}
+                                                                    alt='AI'
+                                                                    width='25'
+                                                                    height='25'
+                                                                    className='boticon'
+                                                                />
+                                                            ) : (
+                                                                <img
+                                                                    style={{ marginLeft: '10px' }}
+                                                                    src={userPNG}
+                                                                    alt='Me'
+                                                                    width='25'
+                                                                    height='25'
+                                                                    className='usericon'
+                                                                />
+                                                            )}
+                                                            <div
+                                                                style={{
+                                                                    display: 'flex',
+                                                                    flexDirection: 'column',
+                                                                    width: '100%'
+                                                                }}
+                                                            >
+                                                                <div className='markdownanswer'>
+                                                                    {/* Messages are being rendered in Markdown format */}
+                                                                    <MemoizedReactMarkdown
+                                                                        remarkPlugins={[remarkGfm, remarkMath]}
+                                                                        rehypePlugins={[rehypeMathjax]}
+                                                                        components={{
+                                                                            code({ inline, className, children, ...props }) {
+                                                                                const match = /language-(\w+)/.exec(className || '')
+                                                                                return !inline ? (
+                                                                                    <CodeBlock
+                                                                                        key={Math.random()}
+                                                                                        chatflowid={dialogProps.chatflow.id}
+                                                                                        isDialog={true}
+                                                                                        language={(match && match[1]) || ''}
+                                                                                        value={String(children).replace(/\n$/, '')}
+                                                                                        {...props}
+                                                                                    />
+                                                                                ) : (
+                                                                                    <code className={className} {...props}>
+                                                                                        {children}
+                                                                                    </code>
+                                                                                )
+                                                                            }
+                                                                        }}
+                                                                    >
+                                                                        {message.message}
+                                                                    </MemoizedReactMarkdown>
+                                                                </div>
+                                                                {message.sourceDocuments && (
+                                                                    <div style={{ display: 'block', flexDirection: 'row', width: '100%' }}>
+                                                                        {removeDuplicateURL(message).map((source, index) => {
+                                                                            const URL = isValidURL(source.metadata.source)
+                                                                            return (
+                                                                                <Chip
+                                                                                    size='small'
+                                                                                    key={index}
+                                                                                    label={
+                                                                                        URL
+                                                                                            ? URL.pathname.substring(0, 15) === '/'
+                                                                                                ? URL.host
+                                                                                                : `${URL.pathname.substring(0, 15)}...`
+                                                                                            : `${source.pageContent.substring(0, 15)}...`
+                                                                                    }
+                                                                                    component='a'
+                                                                                    sx={{ mr: 1, mb: 1 }}
+                                                                                    variant='outlined'
+                                                                                    clickable
+                                                                                    onClick={() =>
+                                                                                        URL
+                                                                                            ? onURLClick(source.metadata.source)
+                                                                                            : onSourceDialogClick(source)
+                                                                                    }
+                                                                                />
+                                                                            )
+                                                                        })}
+                                                                    </div>
+                                                                )}
+                                                            </div>
+                                                        </Box>
+                                                    )
+                                                } else {
+                                                    return (
+                                                        <Box
+                                                            sx={{
+                                                                background: theme.palette.timeMessage.main,
+                                                                p: 2
+                                                            }}
+                                                            key={index}
+                                                            style={{ display: 'flex', justifyContent: 'center', alignContent: 'center' }}
+                                                        >
+                                                            {moment(message.message).format('MMMM Do YYYY, h:mm:ss a')}
+                                                        </Box>
+                                                    )
+                                                }
+                                            })}
+                                    </div>
+                                </div>
+                            </div>
+                        )}
+                    </div>
+                    <SourceDocDialog show={sourceDialogOpen} dialogProps={sourceDialogProps} onCancel={() => setSourceDialogOpen(false)} />
+                </>
+            </DialogContent>
+        </Dialog>
+    ) : null
+
+    return createPortal(component, portalElement)
+}
+
+ViewMessagesDialog.propTypes = {
+    show: PropTypes.bool,
+    dialogProps: PropTypes.object,
+    onCancel: PropTypes.func
+}
+
+export default ViewMessagesDialog

packages/ui/src/ui-component/dropdown/MultiDropdown.js

@@ -18,7 +18,7 @@ const StyledPopper = styled(Popper)({
     }
 })
 
-export const MultiDropdown = ({ name, value, options, onSelect, disabled = false, disableClearable = false }) => {
+export const MultiDropdown = ({ name, value, options, onSelect, formControlSx = {}, disabled = false, disableClearable = false }) => {
     const customization = useSelector((state) => state.customization)
     const findMatchingOptions = (options = [], internalValue) => {
         let values = []
@@ -30,7 +30,7 @@ export const MultiDropdown = ({ name, value, options, onSelect, disabled = false
     let [internalValue, setInternalValue] = useState(value ?? [])
 
     return (
-        <FormControl sx={{ mt: 1, width: '100%' }} size='small'>
+        <FormControl sx={{ mt: 1, width: '100%', ...formControlSx }} size='small'>
             <Autocomplete
                 id={name}
                 disabled={disabled}
@@ -75,5 +75,6 @@ MultiDropdown.propTypes = {
     options: PropTypes.array,
     onSelect: PropTypes.func,
     disabled: PropTypes.bool,
+    formControlSx: PropTypes.object,
     disableClearable: PropTypes.bool
 }

packages/ui/src/ui-component/input/Input.js

@@ -1,10 +1,39 @@
-import { useState } from 'react'
+import { useState, useEffect, useRef } from 'react'
 import PropTypes from 'prop-types'
-import { FormControl, OutlinedInput } from '@mui/material'
+import { FormControl, OutlinedInput, Popover } from '@mui/material'
 import ExpandTextDialog from 'ui-component/dialog/ExpandTextDialog'
+import SelectVariable from 'ui-component/json/SelectVariable'
+import { getAvailableNodesForVariable } from 'utils/genericHelper'
 
-export const Input = ({ inputParam, value, onChange, disabled = false, showDialog, dialogProps, onDialogCancel, onDialogConfirm }) => {
+export const Input = ({
+    inputParam,
+    value,
+    nodes,
+    edges,
+    nodeId,
+    onChange,
+    disabled = false,
+    showDialog,
+    dialogProps,
+    onDialogCancel,
+    onDialogConfirm
+}) => {
     const [myValue, setMyValue] = useState(value ?? '')
+    const [anchorEl, setAnchorEl] = useState(null)
+    const [availableNodesForVariable, setAvailableNodesForVariable] = useState([])
+    const ref = useRef(null)
+
+    const openPopOver = Boolean(anchorEl)
+
+    const handleClosePopOver = () => {
+        setAnchorEl(null)
+    }
+
+    const setNewVal = (val) => {
+        const newVal = myValue + val.substring(2)
+        onChange(newVal)
+        setMyValue(newVal)
+    }
 
     const getInputType = (type) => {
         switch (type) {
@@ -19,6 +48,19 @@ export const Input = ({ inputParam, value, onChange, disabled = false, showDialo
         }
     }
 
+    useEffect(() => {
+        if (!disabled && nodes && edges && nodeId && inputParam) {
+            const nodesForVariable = inputParam?.acceptVariable ? getAvailableNodesForVariable(nodes, edges, nodeId, inputParam.id) : []
+            setAvailableNodesForVariable(nodesForVariable)
+        }
+    }, [disabled, inputParam, nodes, edges, nodeId])
+
+    useEffect(() => {
+        if (typeof myValue === 'string' && myValue && myValue.endsWith('{{')) {
+            setAnchorEl(ref.current)
+        }
+    }, [myValue])
+
     return (
         <>
             <FormControl sx={{ mt: 1, width: '100%' }} size='small'>
@@ -55,6 +97,31 @@ export const Input = ({ inputParam, value, onChange, disabled = false, showDialo
                     }}
                 ></ExpandTextDialog>
             )}
+            <div ref={ref}></div>
+            {inputParam?.acceptVariable && (
+                <Popover
+                    open={openPopOver}
+                    anchorEl={anchorEl}
+                    onClose={handleClosePopOver}
+                    anchorOrigin={{
+                        vertical: 'bottom',
+                        horizontal: 'left'
+                    }}
+                    transformOrigin={{
+                        vertical: 'top',
+                        horizontal: 'left'
+                    }}
+                >
+                    <SelectVariable
+                        disabled={disabled}
+                        availableNodesForVariable={availableNodesForVariable}
+                        onSelectAndReturnVal={(val) => {
+                            setNewVal(val)
+                            handleClosePopOver()
+                        }}
+                    />
+                </Popover>
+            )}
         </>
     )
 }
@@ -66,6 +133,9 @@ Input.propTypes = {
     disabled: PropTypes.bool,
     showDialog: PropTypes.bool,
     dialogProps: PropTypes.object,
+    nodes: PropTypes.array,
+    edges: PropTypes.array,
+    nodeId: PropTypes.string,
     onDialogCancel: PropTypes.func,
     onDialogConfirm: PropTypes.func
 }

packages/ui/src/utils/genericHelper.js

@@ -416,6 +416,23 @@ export const getInputVariables = (paramValue) => {
     return inputVariables
 }
 
+export const removeDuplicateURL = (message) => {
+    const visitedURLs = []
+    const newSourceDocuments = []
+
+    if (!message.sourceDocuments) return newSourceDocuments
+
+    message.sourceDocuments.forEach((source) => {
+        if (isValidURL(source.metadata.source) && !visitedURLs.includes(source.metadata.source)) {
+            visitedURLs.push(source.metadata.source)
+            newSourceDocuments.push(source)
+        } else if (!isValidURL(source.metadata.source)) {
+            newSourceDocuments.push(source)
+        }
+    })
+    return newSourceDocuments
+}
+
 export const isValidURL = (url) => {
     try {
         return new URL(url)

packages/ui/src/views/canvas/CanvasHeader.js

@@ -15,6 +15,7 @@ import Settings from 'views/settings'
 import SaveChatflowDialog from 'ui-component/dialog/SaveChatflowDialog'
 import APICodeDialog from 'views/chatflows/APICodeDialog'
 import AnalyseFlowDialog from 'ui-component/dialog/AnalyseFlowDialog'
+import ViewMessagesDialog from 'ui-component/dialog/ViewMessagesDialog'
 
 // API
 import chatflowsApi from 'api/chatflows'
@@ -44,6 +45,8 @@ const CanvasHeader = ({ chatflow, handleSaveFlow, handleDeleteFlow, handleLoadFl
     const [apiDialogProps, setAPIDialogProps] = useState({})
     const [analyseDialogOpen, setAnalyseDialogOpen] = useState(false)
     const [analyseDialogProps, setAnalyseDialogProps] = useState({})
+    const [viewMessagesDialogOpen, setViewMessagesDialogOpen] = useState(false)
+    const [viewMessagesDialogProps, setViewMessagesDialogProps] = useState({})
 
     const updateChatflowApi = useApi(chatflowsApi.updateChatflow)
     const canvas = useSelector((state) => state.canvas)
@@ -59,6 +62,12 @@ const CanvasHeader = ({ chatflow, handleSaveFlow, handleDeleteFlow, handleLoadFl
                 chatflow: chatflow
             })
             setAnalyseDialogOpen(true)
+        } else if (setting === 'viewMessages') {
+            setViewMessagesDialogProps({
+                title: 'View Messages',
+                chatflow: chatflow
+            })
+            setViewMessagesDialogOpen(true)
         } else if (setting === 'duplicateChatflow') {
             try {
                 localStorage.setItem('duplicatedFlowData', chatflow.flowData)
@@ -69,7 +78,7 @@ const CanvasHeader = ({ chatflow, handleSaveFlow, handleDeleteFlow, handleLoadFl
         } else if (setting === 'exportChatflow') {
             try {
                 const flowData = JSON.parse(chatflow.flowData)
-                let dataStr = JSON.stringify(generateExportFlowData(flowData))
+                let dataStr = JSON.stringify(generateExportFlowData(flowData), null, 2)
                 let dataUri = 'data:application/json;charset=utf-8,' + encodeURIComponent(dataStr)
 
                 let exportFileDefaultName = `${chatflow.name} Chatflow.json`
@@ -367,6 +376,11 @@ const CanvasHeader = ({ chatflow, handleSaveFlow, handleDeleteFlow, handleLoadFl
             />
             <APICodeDialog show={apiDialogOpen} dialogProps={apiDialogProps} onCancel={() => setAPIDialogOpen(false)} />
             <AnalyseFlowDialog show={analyseDialogOpen} dialogProps={analyseDialogProps} onCancel={() => setAnalyseDialogOpen(false)} />
+            <ViewMessagesDialog
+                show={viewMessagesDialogOpen}
+                dialogProps={viewMessagesDialogProps}
+                onCancel={() => setViewMessagesDialogOpen(false)}
+            />
         </>
     )
 }

packages/ui/src/views/canvas/NodeInputHandler.js

@@ -275,6 +275,9 @@ const NodeInputHandler = ({ inputAnchor, inputParam, data, disabled = false, isA
                                 inputParam={inputParam}
                                 onChange={(newValue) => (data.inputs[inputParam.name] = newValue)}
                                 value={data.inputs[inputParam.name] ?? inputParam.default ?? ''}
+                                nodes={inputParam?.acceptVariable && reactFlowInstance ? reactFlowInstance.getNodes() : []}
+                                edges={inputParam?.acceptVariable && reactFlowInstance ? reactFlowInstance.getEdges() : []}
+                                nodeId={data.id}
                                 showDialog={showExpandDialog}
                                 dialogProps={expandDialogProps}
                                 onDialogCancel={() => setShowExpandDialog(false)}

packages/ui/src/views/chatflows/ShareChatbot.js

@@ -57,6 +57,9 @@ const ShareChatbot = ({ isSessionMemory }) => {
     const [isPublicChatflow, setChatflowIsPublic] = useState(chatflow.isPublic ?? false)
     const [generateNewSession, setGenerateNewSession] = useState(chatbotConfig?.generateNewSession ?? false)
 
+    const [title, setTitle] = useState(chatbotConfig?.title ?? '')
+    const [titleAvatarSrc, setTitleAvatarSrc] = useState(chatbotConfig?.titleAvatarSrc ?? '')
+
     const [welcomeMessage, setWelcomeMessage] = useState(chatbotConfig?.welcomeMessage ?? '')
     const [backgroundColor, setBackgroundColor] = useState(chatbotConfig?.backgroundColor ?? defaultConfig.backgroundColor)
     const [fontSize, setFontSize] = useState(chatbotConfig?.fontSize ?? defaultConfig.fontSize)
@@ -108,6 +111,8 @@ const ShareChatbot = ({ isSessionMemory }) => {
             textInput: {},
             overrideConfig: {}
         }
+        if (title) obj.title = title
+        if (titleAvatarSrc) obj.titleAvatarSrc = titleAvatarSrc
         if (welcomeMessage) obj.welcomeMessage = welcomeMessage
         if (backgroundColor) obj.backgroundColor = backgroundColor
         if (fontSize) obj.fontSize = fontSize
@@ -252,6 +257,12 @@ const ShareChatbot = ({ isSessionMemory }) => {
 
     const onTextChanged = (value, fieldName) => {
         switch (fieldName) {
+            case 'title':
+                setTitle(value)
+                break
+            case 'titleAvatarSrc':
+                setTitleAvatarSrc(value)
+                break
             case 'welcomeMessage':
                 setWelcomeMessage(value)
                 break
@@ -395,6 +406,14 @@ const ShareChatbot = ({ isSessionMemory }) => {
                     />
                 </div>
             </Stack>
+            {textField(title, 'title', 'Title', 'string', 'Flowise Assistant')}
+            {textField(
+                titleAvatarSrc,
+                'titleAvatarSrc',
+                'Title Avatar Link',
+                'string',
+                `https://raw.githubusercontent.com/FlowiseAI/Flowise/main/assets/FloWiseAI_dark.png`
+            )}
             {textField(welcomeMessage, 'welcomeMessage', 'Welcome Message', 'string', 'Hello! This is custom welcome message')}
             {colorField(backgroundColor, 'backgroundColor', 'Background Color')}
             {textField(fontSize, 'fontSize', 'Font Size', 'number')}

packages/ui/src/views/chatmessage/ChatMessage.css

@@ -134,3 +134,13 @@
     justify-content: center;
     align-items: center;
 }
+
+.cloud-message {
+    width: 100%;
+    height: calc(100vh - 260px);
+    overflow-y: scroll;
+    border-radius: 0.5rem;
+    display: flex;
+    justify-content: center;
+    align-items: center;
+}

packages/ui/src/views/chatmessage/ChatMessage.js

@@ -30,7 +30,7 @@ import { baseURL, maxScroll } from 'store/constant'
 
 import robotPNG from 'assets/images/robot.png'
 import userPNG from 'assets/images/account.png'
-import { isValidURL } from 'utils/genericHelper'
+import { isValidURL, removeDuplicateURL } from 'utils/genericHelper'
 
 export const ChatMessage = ({ open, chatflowid, isDialog }) => {
     const theme = useTheme()
@@ -50,9 +50,10 @@ export const ChatMessage = ({ open, chatflowid, isDialog }) => {
     const [isChatFlowAvailableToStream, setIsChatFlowAvailableToStream] = useState(false)
     const [sourceDialogOpen, setSourceDialogOpen] = useState(false)
     const [sourceDialogProps, setSourceDialogProps] = useState({})
+    const [chatId, setChatId] = useState(undefined)
 
     const inputRef = useRef(null)
-    const getChatmessageApi = useApi(chatmessageApi.getChatmessageFromChatflow)
+    const getChatmessageApi = useApi(chatmessageApi.getInternalChatmessageFromChatflow)
     const getIsChatflowStreamingApi = useApi(chatflowsApi.getIsChatflowStreaming)
 
     const onSourceDialogClick = (data) => {
@@ -64,21 +65,6 @@ export const ChatMessage = ({ open, chatflowid, isDialog }) => {
         window.open(data, '_blank')
     }
 
-    const removeDuplicateURL = (message) => {
-        const visitedURLs = []
-        const newSourceDocuments = []
-
-        message.sourceDocuments.forEach((source) => {
-            if (isValidURL(source.metadata.source) && !visitedURLs.includes(source.metadata.source)) {
-                visitedURLs.push(source.metadata.source)
-                newSourceDocuments.push(source)
-            } else if (!isValidURL(source.metadata.source)) {
-                newSourceDocuments.push(source)
-            }
-        })
-        return newSourceDocuments
-    }
-
     const scrollToBottom = () => {
         if (ps.current) {
             ps.current.scrollTo({ top: maxScroll })
@@ -87,20 +73,6 @@ export const ChatMessage = ({ open, chatflowid, isDialog }) => {
 
     const onChange = useCallback((e) => setUserInput(e.target.value), [setUserInput])
 
-    const addChatMessage = async (message, type, sourceDocuments) => {
-        try {
-            const newChatMessageBody = {
-                role: type,
-                content: message,
-                chatflowid: chatflowid
-            }
-            if (sourceDocuments) newChatMessageBody.sourceDocuments = JSON.stringify(sourceDocuments)
-            await chatmessageApi.createNewChatmessage(chatflowid, newChatMessageBody)
-        } catch (error) {
-            console.error(error)
-        }
-    }
-
     const updateLastMessage = (text) => {
         setMessages((prevMessages) => {
             let allMessages = [...cloneDeep(prevMessages)]
@@ -123,7 +95,6 @@ export const ChatMessage = ({ open, chatflowid, isDialog }) => {
     const handleError = (message = 'Oops! There seems to be an error. Please try again.') => {
         message = message.replace(`Unable to parse JSON response from chat agent.\n\n`, '')
         setMessages((prevMessages) => [...prevMessages, { message, type: 'apiMessage' }])
-        addChatMessage(message, 'apiMessage')
         setLoading(false)
         setUserInput('')
         setTimeout(() => {
@@ -141,42 +112,36 @@ export const ChatMessage = ({ open, chatflowid, isDialog }) => {
 
         setLoading(true)
         setMessages((prevMessages) => [...prevMessages, { message: userInput, type: 'userMessage' }])
-        // waiting for first chatmessage saved, the first chatmessage will be used in sendMessageAndGetPrediction
-        await addChatMessage(userInput, 'userMessage')
 
         // Send user question and history to API
         try {
             const params = {
                 question: userInput,
-                history: messages.filter((msg) => msg.message !== 'Hi there! How can I help?')
+                history: messages.filter((msg) => msg.message !== 'Hi there! How can I help?'),
+                chatId
             }
             if (isChatFlowAvailableToStream) params.socketIOClientId = socketIOClientId
 
             const response = await predictionApi.sendMessageAndGetPrediction(chatflowid, params)
 
             if (response.data) {
-                let data = response.data
+                const data = response.data
-
+                if (!chatId) {
-                if (typeof data === 'object' && data.text && data.sourceDocuments) {
+                    setChatId(data.chatId)
-                    if (!isChatFlowAvailableToStream) {
+                    localStorage.setItem(`${chatflowid}_INTERNAL`, data.chatId)
-                        setMessages((prevMessages) => [
-                            ...prevMessages,
-                            { message: data.text, sourceDocuments: data.sourceDocuments, type: 'apiMessage' }
-                        ])
-                    }
-                    addChatMessage(data.text, 'apiMessage', data.sourceDocuments)
-                } else if (typeof data === 'object' && data.json) {
-                    const text = '```json\n' + JSON.stringify(data.json, null, 2)
-                    if (!isChatFlowAvailableToStream) {
-                        setMessages((prevMessages) => [...prevMessages, { message: text, type: 'apiMessage' }])
-                    }
-                    addChatMessage(text, 'apiMessage')
-                } else {
-                    if (!isChatFlowAvailableToStream) {
-                        setMessages((prevMessages) => [...prevMessages, { message: data, type: 'apiMessage' }])
-                    }
-                    addChatMessage(data, 'apiMessage')
                 }
+                if (!isChatFlowAvailableToStream) {
+                    let text = ''
+                    if (data.text) text = data.text
+                    else if (data.json) text = '```json\n' + JSON.stringify(data.json, null, 2)
+                    else text = JSON.stringify(data, null, 2)
+
+                    setMessages((prevMessages) => [
+                        ...prevMessages,
+                        { message: text, sourceDocuments: data?.sourceDocuments, type: 'apiMessage' }
+                    ])
+                }
+
                 setLoading(false)
                 setUserInput('')
                 setTimeout(() => {
@@ -206,16 +171,18 @@ export const ChatMessage = ({ open, chatflowid, isDialog }) => {
 
     // Get chatmessages successful
     useEffect(() => {
-        if (getChatmessageApi.data) {
+        if (getChatmessageApi.data?.length) {
-            const loadedMessages = []
+            const chatId = getChatmessageApi.data[0]?.chatId
-            for (const message of getChatmessageApi.data) {
+            setChatId(chatId)
+            localStorage.setItem(`${chatflowid}_INTERNAL`, chatId)
+            const loadedMessages = getChatmessageApi.data.map((message) => {
                 const obj = {
                     message: message.content,
                     type: message.role
                 }
                 if (message.sourceDocuments) obj.sourceDocuments = JSON.parse(message.sourceDocuments)
-                loadedMessages.push(obj)
+                return obj
-            }
+            })
             setMessages((prevMessages) => [...prevMessages, ...loadedMessages])
         }
 

packages/ui/src/views/chatmessage/ChatPopUp.js

@@ -85,7 +85,9 @@ export const ChatPopUp = ({ chatflowid }) => {
 
         if (isConfirmed) {
             try {
-                await chatmessageApi.deleteChatmessage(chatflowid)
+                const chatId = localStorage.getItem(`${chatflowid}_INTERNAL`)
+                await chatmessageApi.deleteChatmessage(chatflowid, { chatId, chatType: 'INTERNAL' })
+                localStorage.removeItem(`${chatflowid}_INTERNAL`)
                 resetChatDialog()
                 enqueueSnackbar({
                     message: 'Succesfully cleared all chat history',

packages/ui/src/views/tools/ToolDialog.js

@@ -213,7 +213,7 @@ const ToolDialog = ({ show, dialogProps, onUseTemplate, onCancel, onConfirm }) =
                 delete toolData.id
                 delete toolData.createdDate
                 delete toolData.updatedDate
-                let dataStr = JSON.stringify(toolData)
+                let dataStr = JSON.stringify(toolData, null, 2)
                 let dataUri = 'data:application/json;charset=utf-8,' + encodeURIComponent(dataStr)
 
                 let exportFileDefaultName = `${toolName}-CustomTool.json`