Merge branch 'main' into feature/ChatHistory2

packages/components/credentials/SearchApi.credential.ts

@@ -0,0 +1,26 @@
+import { INodeParams, INodeCredential } from '../src/Interface'
+
+class SearchApi implements INodeCredential {
+    label: string
+    name: string
+    version: number
+    description: string
+    inputs: INodeParams[]
+
+    constructor() {
+        this.label = 'Search API'
+        this.name = 'searchApi'
+        this.version = 1.0
+        this.description =
+            'Sign in to <a target="_blank" href="https://www.searchapi.io/">SearchApi</a> to obtain a free API key from the dashboard.'
+        this.inputs = [
+            {
+                label: 'SearchApi API Key',
+                name: 'searchApiKey',
+                type: 'password'
+            }
+        ]
+    }
+}
+
+module.exports = { credClass: SearchApi }

packages/components/nodes/agents/CSVAgent/CSVAgent.ts

@@ -131,7 +131,7 @@ json.dumps(my_dict)`
                 const code = `import pandas as pd\n${pythonCode}`
                 finalResult = await pyodide.runPythonAsync(code)
             } catch (error) {
-                throw new Error(`Sorry, I'm unable to find answer for question: "${input}" using follwoing code: "${pythonCode}"`)
+                throw new Error(`Sorry, I'm unable to find answer for question: "${input}" using following code: "${pythonCode}"`)
             }
         }
 

packages/components/nodes/chains/ConversationChain/ConversationChain.ts

@@ -71,7 +71,9 @@ class ConversationChain_Chains implements INode {
         const flattenDocs = docs && docs.length ? flatten(docs) : []
         const finalDocs = []
         for (let i = 0; i < flattenDocs.length; i += 1) {
-            finalDocs.push(new Document(flattenDocs[i]))
+            if (flattenDocs[i] && flattenDocs[i].pageContent) {
+                finalDocs.push(new Document(flattenDocs[i]))
+            }
         }
 
         let finalText = ''

packages/components/nodes/documentloaders/SearchApi/SearchAPI.ts

@@ -0,0 +1,109 @@
+import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
+import { TextSplitter } from 'langchain/text_splitter'
+import { SearchApiLoader } from 'langchain/document_loaders/web/searchapi'
+import { getCredentialData, getCredentialParam } from '../../../src'
+
+// Provides access to multiple search engines using the SearchApi.
+// For available parameters & engines, refer to: https://www.searchapi.io/docs/google
+class SearchAPI_DocumentLoaders implements INode {
+    label: string
+    name: string
+    version: number
+    description: string
+    type: string
+    icon: string
+    category: string
+    baseClasses: string[]
+    credential: INodeParams
+    inputs: INodeParams[]
+
+    constructor() {
+        this.label = 'SearchApi For Web Search'
+        this.name = 'searchApi'
+        this.version = 1.0
+        this.type = 'Document'
+        this.icon = 'searchapi.svg'
+        this.category = 'Document Loaders'
+        this.description = 'Load data from real-time search results'
+        this.baseClasses = [this.type]
+        this.credential = {
+            label: 'Connect Credential',
+            name: 'credential',
+            type: 'credential',
+            optional: false,
+            credentialNames: ['searchApi']
+        }
+        this.inputs = [
+            {
+                label: 'Query',
+                name: 'query',
+                type: 'string',
+                optional: true
+            },
+            {
+                label: 'Custom Parameters',
+                name: 'customParameters',
+                type: 'json',
+                optional: true,
+                additionalParams: true
+            },
+            {
+                label: 'Text Splitter',
+                name: 'textSplitter',
+                type: 'TextSplitter',
+                optional: true
+            },
+            {
+                label: 'Metadata',
+                name: 'metadata',
+                type: 'json',
+                optional: true,
+                additionalParams: true
+            }
+        ]
+    }
+
+    async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
+        const textSplitter = nodeData.inputs?.textSplitter as TextSplitter
+        const query = nodeData.inputs?.query as string
+        const customParameters = nodeData.inputs?.customParameters
+        const metadata = nodeData.inputs?.metadata
+
+        // Fetch the API credentials for this node
+        const credentialData = await getCredentialData(nodeData.credential ?? '', options)
+        const searchApiKey = getCredentialParam('searchApiKey', credentialData, nodeData)
+
+        // Check and parse custom parameters (should be JSON or object)
+        const parsedParameters = typeof customParameters === 'object' ? customParameters : JSON.parse(customParameters || '{}')
+
+        // Prepare the configuration for the SearchApiLoader
+        const loaderConfig = {
+            q: query,
+            apiKey: searchApiKey,
+            ...parsedParameters
+        }
+
+        // Initialize the loader with the given configuration
+        const loader = new SearchApiLoader(loaderConfig)
+
+        // Fetch documents, split if a text splitter is provided
+        const docs = textSplitter ? await loader.loadAndSplit() : await loader.load()
+
+        if (metadata) {
+            const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata)
+            return docs.map((doc) => {
+                return {
+                    ...doc,
+                    metadata: {
+                        ...doc.metadata,
+                        ...parsedMetadata
+                    }
+                }
+            })
+        }
+
+        return docs
+    }
+}
+
+module.exports = { nodeClass: SearchAPI_DocumentLoaders }

packages/components/nodes/documentloaders/SearchApi/searchapi.svg

@@ -0,0 +1 @@
+<svg id="SvgjsSvg1001" width="75.27131652832031" height="63.92396926879883" xmlns="http://www.w3.org/2000/svg" version="1.1" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:svgjs="http://svgjs.com/svgjs" viewBox="0 0 75.27131652832031 63.92396926879883"><defs id="SvgjsDefs1002"></defs><rect id="SvgjsRect1008" width="75.27131652832031" height="63.92396926879883" fill="transparent"></rect><g id="SvgjsG1009" transform="matrix(1,0,0,1,-39.50003433227539,-50.53549575805664)"><title>0479_octopus_verti</title><path id="color_1" d="M97.24234,109.8245a2.57759,2.57759,0,0,1-2.57778,2.57778,9.80672,9.80672,0,0,1-9.79558-9.79557V91.58366a2.57779,2.57779,0,0,1,5.15557,0v11.02305a4.64509,4.64509,0,0,0,4.64,4.64A2.57759,2.57759,0,0,1,97.24234,109.8245ZM112.19348,93.223a2.57759,2.57759,0,0,0-2.57778,2.57779,4.64,4.64,0,1,1-9.28,0V73.73554a23.2,23.2,0,1,0-46.40009,0V95.80076a4.64,4.64,0,1,1-9.28,0,2.57779,2.57779,0,1,0-5.15557,0,9.79558,9.79558,0,0,0,19.59115,0V73.73554a18.04449,18.04449,0,0,1,36.089,0V95.80076a9.79558,9.79558,0,0,0,19.59115,0A2.57759,2.57759,0,0,0,112.19348,93.223ZM77.13563,91.78a2.57759,2.57759,0,0,0-2.57778,2.57778v17.52893a2.57779,2.57779,0,0,0,5.15557,0V94.3578A2.57759,2.57759,0,0,0,77.13563,91.78ZM66.8245,89.00588a2.57759,2.57759,0,0,0-2.57778,2.57778v11.02305a4.64509,4.64509,0,0,1-4.64,4.64,2.57778,2.57778,0,1,0,0,5.15556,9.80671,9.80671,0,0,0,9.79557-9.79557V91.58366A2.57759,2.57759,0,0,0,66.8245,89.00588ZM69.918,70.12639a3.6089,3.6089,0,1,0,3.6089,3.6089A3.60891,3.60891,0,0,0,69.918,70.12639Zm18.04448,3.6089a3.6089,3.6089,0,1,0-3.60889,3.60889A3.60891,3.60891,0,0,0,87.96251,73.73529Z" fill="#3730a3"></path></g></svg>

packages/components/nodes/documentloaders/Text/Text.ts

@@ -19,7 +19,7 @@ class Text_DocumentLoaders implements INode {
     constructor() {
         this.label = 'Text File'
         this.name = 'textFile'
-        this.version = 2.0
+        this.version = 3.0
         this.type = 'Document'
         this.icon = 'textFile.svg'
         this.category = 'Document Loaders'
@@ -30,7 +30,8 @@ class Text_DocumentLoaders implements INode {
                 label: 'Txt File',
                 name: 'txtFile',
                 type: 'file',
-                fileType: '.txt'
+                fileType:
+                    '.txt, .html, .aspx, .asp, .cpp, .c, .cs, .css, .go, .h, .java, .js, .less, .ts, .php, .proto, .python, .py, .rst, .ruby, .rb, .rs, .scala, .sc, .scss, .sol, .sql, .swift, .markdown, .md, .tex, .ltx, .vb, .xml'
             },
             {
                 label: 'Text Splitter',

packages/components/nodes/documentloaders/VectorStoreToDocument/VectorStoreToDocument.ts

@@ -17,7 +17,7 @@ class VectorStoreToDocument_DocumentLoaders implements INode {
     constructor() {
         this.label = 'VectorStore To Document'
         this.name = 'vectorStoreToDocument'
-        this.version = 1.0
+        this.version = 2.0
         this.type = 'Document'
         this.icon = 'vectorretriever.svg'
         this.category = 'Document Loaders'
@@ -29,6 +29,14 @@ class VectorStoreToDocument_DocumentLoaders implements INode {
                 name: 'vectorStore',
                 type: 'VectorStore'
             },
+            {
+                label: 'Query',
+                name: 'query',
+                type: 'string',
+                description: 'Query to retrieve documents from vector database. If not specified, user question will be used',
+                optional: true,
+                acceptVariable: true
+            },
             {
                 label: 'Minimum Score (%)',
                 name: 'minScore',
@@ -56,11 +64,12 @@ class VectorStoreToDocument_DocumentLoaders implements INode {
     async init(nodeData: INodeData, input: string): Promise<any> {
         const vectorStore = nodeData.inputs?.vectorStore as VectorStore
         const minScore = nodeData.inputs?.minScore as number
+        const query = nodeData.inputs?.query as string
         const output = nodeData.outputs?.output as string
 
         const topK = (vectorStore as any)?.k ?? 4
 
-        const docs = await vectorStore.similaritySearchWithScore(input, topK)
+        const docs = await vectorStore.similaritySearchWithScore(query ?? input, topK)
         // eslint-disable-next-line no-console
         console.log('\x1b[94m\x1b[1m\n*****VectorStore Documents*****\n\x1b[0m\x1b[0m')
         // eslint-disable-next-line no-console

packages/components/nodes/embeddings/OpenAIEmbeddingCustom/OpenAIEmbeddingCustom.ts

@@ -0,0 +1,94 @@
+import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
+import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
+import { OpenAIEmbeddings, OpenAIEmbeddingsParams } from 'langchain/embeddings/openai'
+
+class OpenAIEmbeddingCustom_Embeddings implements INode {
+    label: string
+    name: string
+    version: number
+    type: string
+    icon: string
+    category: string
+    description: string
+    baseClasses: string[]
+    credential: INodeParams
+    inputs: INodeParams[]
+
+    constructor() {
+        this.label = 'OpenAI Embeddings Custom'
+        this.name = 'openAIEmbeddingsCustom'
+        this.version = 1.0
+        this.type = 'OpenAIEmbeddingsCustom'
+        this.icon = 'openai.png'
+        this.category = 'Embeddings'
+        this.description = 'OpenAI API to generate embeddings for a given text'
+        this.baseClasses = [this.type, ...getBaseClasses(OpenAIEmbeddings)]
+        this.credential = {
+            label: 'Connect Credential',
+            name: 'credential',
+            type: 'credential',
+            credentialNames: ['openAIApi']
+        }
+        this.inputs = [
+            {
+                label: 'Strip New Lines',
+                name: 'stripNewLines',
+                type: 'boolean',
+                optional: true,
+                additionalParams: true
+            },
+            {
+                label: 'Batch Size',
+                name: 'batchSize',
+                type: 'number',
+                optional: true,
+                additionalParams: true
+            },
+            {
+                label: 'Timeout',
+                name: 'timeout',
+                type: 'number',
+                optional: true,
+                additionalParams: true
+            },
+            {
+                label: 'BasePath',
+                name: 'basepath',
+                type: 'string',
+                optional: true,
+                additionalParams: true
+            },
+            {
+                label: 'Model Name',
+                name: 'modelName',
+                type: 'string',
+                optional: true
+            }
+        ]
+    }
+
+    async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
+        const stripNewLines = nodeData.inputs?.stripNewLines as boolean
+        const batchSize = nodeData.inputs?.batchSize as string
+        const timeout = nodeData.inputs?.timeout as string
+        const basePath = nodeData.inputs?.basepath as string
+        const modelName = nodeData.inputs?.modelName as string
+
+        const credentialData = await getCredentialData(nodeData.credential ?? '', options)
+        const openAIApiKey = getCredentialParam('openAIApiKey', credentialData, nodeData)
+
+        const obj: Partial<OpenAIEmbeddingsParams> & { openAIApiKey?: string } = {
+            openAIApiKey
+        }
+
+        if (stripNewLines) obj.stripNewLines = stripNewLines
+        if (batchSize) obj.batchSize = parseInt(batchSize, 10)
+        if (timeout) obj.timeout = parseInt(timeout, 10)
+        if (modelName) obj.modelName = modelName
+
+        const model = new OpenAIEmbeddings(obj, { basePath })
+        return model
+    }
+}
+
+module.exports = { nodeClass: OpenAIEmbeddingCustom_Embeddings }

packages/components/nodes/memory/BufferWindowMemory/BufferWindowMemory.ts

@@ -20,7 +20,7 @@ class BufferWindowMemory_Memory implements INode {
         this.type = 'BufferWindowMemory'
         this.icon = 'memory.svg'
         this.category = 'Memory'
-        this.description = 'Uses a window of size k to surface the last k back-and-forths to use as memory'
+        this.description = 'Uses a window of size k to surface the last k back-and-forth to use as memory'
         this.baseClasses = [this.type, ...getBaseClasses(BufferWindowMemory)]
         this.inputs = [
             {
@@ -40,7 +40,7 @@ class BufferWindowMemory_Memory implements INode {
                 name: 'k',
                 type: 'number',
                 default: '4',
-                description: 'Window of size k to surface the last k back-and-forths to use as memory.'
+                description: 'Window of size k to surface the last k back-and-forth to use as memory.'
             }
         ]
     }

packages/components/nodes/memory/RedisBackedChatMemory/RedisBackedChatMemory.ts

@@ -119,27 +119,26 @@ const initalizeRedis = async (nodeData: INodeData, options: ICommonObject): Prom
     const redisChatMessageHistory = new RedisChatMessageHistory(obj)
 
     redisChatMessageHistory.getMessages = async (): Promise<BaseMessage[]> => {
-        const rawStoredMessages = await client.lrange(sessionId ? sessionId : chatId, 0, -1)
+        const rawStoredMessages = await client.lrange((redisChatMessageHistory as any).sessionId, 0, -1)
         const orderedMessages = rawStoredMessages.reverse().map((message) => JSON.parse(message))
         return orderedMessages.map(mapStoredMessageToChatMessage)
     }
 
     redisChatMessageHistory.addMessage = async (message: BaseMessage): Promise<void> => {
         const messageToAdd = [message].map((msg) => msg.toDict())
-        await client.lpush(sessionId ? sessionId : chatId, JSON.stringify(messageToAdd[0]))
+        await client.lpush((redisChatMessageHistory as any).sessionId, JSON.stringify(messageToAdd[0]))
         if (sessionTTL) {
-            await client.expire(sessionId ? sessionId : chatId, sessionTTL)
+            await client.expire((redisChatMessageHistory as any).sessionId, sessionTTL)
         }
     }
 
     redisChatMessageHistory.clear = async (): Promise<void> => {
-        await client.del(sessionId ? sessionId : chatId)
+        await client.del((redisChatMessageHistory as any).sessionId)
     }
 
     const memory = new BufferMemoryExtended({
         memoryKey,
         chatHistory: redisChatMessageHistory,
-        returnMessages: true,
         isSessionIdUsingChatMessageId
     })
     return memory

packages/components/nodes/memory/ZepMemory/ZepMemory.ts

@@ -60,7 +60,7 @@ class ZepMemory_Memory implements INode {
                 name: 'k',
                 type: 'number',
                 default: '10',
-                description: 'Window of size k to surface the last k back-and-forths to use as memory.'
+                description: 'Window of size k to surface the last k back-and-forth to use as memory.'
             },
             {
                 label: 'Auto Summary Template',

packages/components/nodes/prompts/FewShotPromptTemplate/FewShotPromptTemplate.ts

@@ -55,7 +55,7 @@ class FewShotPromptTemplate_Prompts implements INode {
                 placeholder: `Word: {input}\nAntonym:`
             },
             {
-                label: 'Example Seperator',
+                label: 'Example Separator',
                 name: 'exampleSeparator',
                 type: 'string',
                 placeholder: `\n\n`

packages/components/nodes/textsplitters/CharacterTextSplitter/CharacterTextSplitter.ts

@@ -41,7 +41,7 @@ class CharacterTextSplitter_TextSplitters implements INode {
                 name: 'separator',
                 type: 'string',
                 placeholder: `" "`,
-                description: 'Seperator to determine when to split the text, will override the default separator',
+                description: 'Separator to determine when to split the text, will override the default separator',
                 optional: true
             }
         ]

packages/components/nodes/textsplitters/RecursiveCharacterTextSplitter/RecursiveCharacterTextSplitter.ts

@@ -41,8 +41,9 @@ class RecursiveCharacterTextSplitter_TextSplitters implements INode {
                 name: 'separators',
                 type: 'string',
                 rows: 4,
-                description: 'Array of custom seperators to determine when to split the text, will override the default separators',
+                description: 'Array of custom separators to determine when to split the text, will override the default separators',
                 placeholder: `["|", "##", ">", "-"]`,
+                additionalParams: true,
                 optional: true
             }
         ]

packages/components/nodes/tools/SearchApi/SearchAPI.ts

@@ -0,0 +1,42 @@
+import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
+import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
+import { SearchApi } from 'langchain/tools'
+
+class SearchAPI_Tools implements INode {
+    label: string
+    name: string
+    version: number
+    description: string
+    type: string
+    icon: string
+    category: string
+    baseClasses: string[]
+    credential: INodeParams
+    inputs: INodeParams[]
+
+    constructor() {
+        this.label = 'SearchApi'
+        this.name = 'searchAPI'
+        this.version = 1.0
+        this.type = 'SearchAPI'
+        this.icon = 'searchapi.svg'
+        this.category = 'Tools'
+        this.description = 'Real-time API for accessing Google Search data'
+        this.inputs = []
+        this.credential = {
+            label: 'Connect Credential',
+            name: 'credential',
+            type: 'credential',
+            credentialNames: ['searchApi']
+        }
+        this.baseClasses = [this.type, ...getBaseClasses(SearchApi)]
+    }
+
+    async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
+        const credentialData = await getCredentialData(nodeData.credential ?? '', options)
+        const searchApiKey = getCredentialParam('searchApiKey', credentialData, nodeData)
+        return new SearchApi(searchApiKey)
+    }
+}
+
+module.exports = { nodeClass: SearchAPI_Tools }

packages/components/nodes/tools/SearchApi/searchapi.svg

@@ -0,0 +1 @@
+<svg id="SvgjsSvg1001" width="75.27131652832031" height="63.92396926879883" xmlns="http://www.w3.org/2000/svg" version="1.1" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:svgjs="http://svgjs.com/svgjs" viewBox="0 0 75.27131652832031 63.92396926879883"><defs id="SvgjsDefs1002"></defs><rect id="SvgjsRect1008" width="75.27131652832031" height="63.92396926879883" fill="transparent"></rect><g id="SvgjsG1009" transform="matrix(1,0,0,1,-39.50003433227539,-50.53549575805664)"><title>0479_octopus_verti</title><path id="color_1" d="M97.24234,109.8245a2.57759,2.57759,0,0,1-2.57778,2.57778,9.80672,9.80672,0,0,1-9.79558-9.79557V91.58366a2.57779,2.57779,0,0,1,5.15557,0v11.02305a4.64509,4.64509,0,0,0,4.64,4.64A2.57759,2.57759,0,0,1,97.24234,109.8245ZM112.19348,93.223a2.57759,2.57759,0,0,0-2.57778,2.57779,4.64,4.64,0,1,1-9.28,0V73.73554a23.2,23.2,0,1,0-46.40009,0V95.80076a4.64,4.64,0,1,1-9.28,0,2.57779,2.57779,0,1,0-5.15557,0,9.79558,9.79558,0,0,0,19.59115,0V73.73554a18.04449,18.04449,0,0,1,36.089,0V95.80076a9.79558,9.79558,0,0,0,19.59115,0A2.57759,2.57759,0,0,0,112.19348,93.223ZM77.13563,91.78a2.57759,2.57759,0,0,0-2.57778,2.57778v17.52893a2.57779,2.57779,0,0,0,5.15557,0V94.3578A2.57759,2.57759,0,0,0,77.13563,91.78ZM66.8245,89.00588a2.57759,2.57759,0,0,0-2.57778,2.57778v11.02305a4.64509,4.64509,0,0,1-4.64,4.64,2.57778,2.57778,0,1,0,0,5.15556,9.80671,9.80671,0,0,0,9.79557-9.79557V91.58366A2.57759,2.57759,0,0,0,66.8245,89.00588ZM69.918,70.12639a3.6089,3.6089,0,1,0,3.6089,3.6089A3.60891,3.60891,0,0,0,69.918,70.12639Zm18.04448,3.6089a3.6089,3.6089,0,1,0-3.60889,3.60889A3.60891,3.60891,0,0,0,87.96251,73.73529Z" fill="#3730a3"></path></g></svg>

packages/components/nodes/vectorstores/Chroma/Chroma_Upsert.ts

@@ -98,7 +98,9 @@ class ChromaUpsert_VectorStores implements INode {
         const flattenDocs = docs && docs.length ? flatten(docs) : []
         const finalDocs = []
         for (let i = 0; i < flattenDocs.length; i += 1) {
-            finalDocs.push(new Document(flattenDocs[i]))
+            if (flattenDocs[i] && flattenDocs[i].pageContent) {
+                finalDocs.push(new Document(flattenDocs[i]))
+            }
         }
 
         const obj: {

packages/components/nodes/vectorstores/Elasticsearch/Elasticsearch_Upsert.ts

@@ -38,7 +38,9 @@ class ElasicsearchUpsert_VectorStores extends ElasticSearchBase implements INode
         const flattenDocs = docs && docs.length ? flatten(docs) : []
         const finalDocs = []
         for (let i = 0; i < flattenDocs.length; i += 1) {
-            finalDocs.push(new Document(flattenDocs[i]))
+            if (flattenDocs[i] && flattenDocs[i].pageContent) {
+                finalDocs.push(new Document(flattenDocs[i]))
+            }
         }
 
         // The following code is a workaround for a bug (Langchain Issue #1589) in the underlying library.

packages/components/nodes/vectorstores/Faiss_Upsert/Faiss_Upsert.ts

@@ -80,7 +80,9 @@ class FaissUpsert_VectorStores implements INode {
         const flattenDocs = docs && docs.length ? flatten(docs) : []
         const finalDocs = []
         for (let i = 0; i < flattenDocs.length; i += 1) {
-            finalDocs.push(new Document(flattenDocs[i]))
+            if (flattenDocs[i] && flattenDocs[i].pageContent) {
+                finalDocs.push(new Document(flattenDocs[i]))
+            }
         }
 
         const vectorStore = await FaissStore.fromDocuments(finalDocs, embeddings)

packages/components/nodes/vectorstores/InMemory/InMemoryVectorStore.ts

@@ -71,7 +71,9 @@ class InMemoryVectorStore_VectorStores implements INode {
         const flattenDocs = docs && docs.length ? flatten(docs) : []
         const finalDocs = []
         for (let i = 0; i < flattenDocs.length; i += 1) {
-            finalDocs.push(new Document(flattenDocs[i]))
+            if (flattenDocs[i] && flattenDocs[i].pageContent) {
+                finalDocs.push(new Document(flattenDocs[i]))
+            }
         }
 
         const vectorStore = await MemoryVectorStore.fromDocuments(finalDocs, embeddings)

packages/components/nodes/vectorstores/Milvus/Milvus_Upsert.ts

@@ -110,7 +110,9 @@ class Milvus_Upsert_VectorStores implements INode {
         const flattenDocs = docs && docs.length ? flatten(docs) : []
         const finalDocs = []
         for (let i = 0; i < flattenDocs.length; i += 1) {
-            finalDocs.push(new Document(flattenDocs[i]))
+            if (flattenDocs[i] && flattenDocs[i].pageContent) {
+                finalDocs.push(new Document(flattenDocs[i]))
+            }
         }
 
         const vectorStore = await MilvusUpsert.fromDocuments(finalDocs, embeddings, milVusArgs)
@@ -252,7 +254,7 @@ class MilvusUpsert extends Milvus {
             collection_name: this.collectionName
         })
 
-        if (descIndexResp.status.error_code === ErrorCode.INDEX_NOT_EXIST) {
+        if (descIndexResp.status.error_code === ErrorCode.IndexNotExist) {
             const resp = await this.client.createIndex({
                 collection_name: this.collectionName,
                 field_name: this.vectorField,

packages/components/nodes/vectorstores/OpenSearch_Upsert/OpenSearch_Upsert.ts

@@ -86,7 +86,9 @@ class OpenSearchUpsert_VectorStores implements INode {
         const flattenDocs = docs && docs.length ? flatten(docs) : []
         const finalDocs = []
         for (let i = 0; i < flattenDocs.length; i += 1) {
-            finalDocs.push(new Document(flattenDocs[i]))
+            if (flattenDocs[i] && flattenDocs[i].pageContent) {
+                finalDocs.push(new Document(flattenDocs[i]))
+            }
         }
 
         const client = new Client({

packages/components/nodes/vectorstores/Pinecone/Pinecone_Upsert.ts

@@ -106,7 +106,9 @@ class PineconeUpsert_VectorStores implements INode {
         const flattenDocs = docs && docs.length ? flatten(docs) : []
         const finalDocs = []
         for (let i = 0; i < flattenDocs.length; i += 1) {
-            finalDocs.push(new Document(flattenDocs[i]))
+            if (flattenDocs[i] && flattenDocs[i].pageContent) {
+                finalDocs.push(new Document(flattenDocs[i]))
+            }
         }
 
         const obj: PineconeLibArgs = {

packages/components/nodes/vectorstores/Postgres_Upsert/Postgres_Upsert.ts

@@ -143,7 +143,9 @@ class PostgresUpsert_VectorStores implements INode {
         const flattenDocs = docs && docs.length ? flatten(docs) : []
         const finalDocs = []
         for (let i = 0; i < flattenDocs.length; i += 1) {
-            finalDocs.push(new Document(flattenDocs[i]))
+            if (flattenDocs[i] && flattenDocs[i].pageContent) {
+                finalDocs.push(new Document(flattenDocs[i]))
+            }
         }
 
         const vectorStore = await TypeORMVectorStore.fromDocuments(finalDocs, embeddings, args)

packages/components/nodes/vectorstores/Qdrant_Upsert/Qdrant_Upsert.ts

@@ -147,7 +147,9 @@ class QdrantUpsert_VectorStores implements INode {
         const flattenDocs = docs && docs.length ? flatten(docs) : []
         const finalDocs = []
         for (let i = 0; i < flattenDocs.length; i += 1) {
-            finalDocs.push(new Document(flattenDocs[i]))
+            if (flattenDocs[i] && flattenDocs[i].pageContent) {
+                finalDocs.push(new Document(flattenDocs[i]))
+            }
         }
 
         const dbConfig: QdrantLibArgs = {

packages/components/nodes/vectorstores/Redis/Redis_Upsert.ts

@@ -49,9 +49,11 @@ class RedisUpsert_VectorStores extends RedisSearchBase implements INode {
         const flattenDocs = docs && docs.length ? flatten(docs) : []
         const finalDocs = []
         for (let i = 0; i < flattenDocs.length; i += 1) {
-            const document = new Document(flattenDocs[i])
+            if (flattenDocs[i] && flattenDocs[i].pageContent) {
-            escapeAllStrings(document.metadata)
+                const document = new Document(flattenDocs[i])
-            finalDocs.push(document)
+                escapeAllStrings(document.metadata)
+                finalDocs.push(document)
+            }
         }
 
         return super.init(nodeData, _, options, flattenDocs)

packages/components/nodes/vectorstores/Singlestore_Upsert/Singlestore_Upsert.ts

@@ -140,7 +140,9 @@ class SingleStoreUpsert_VectorStores implements INode {
         const flattenDocs = docs && docs.length ? flatten(docs) : []
         const finalDocs = []
         for (let i = 0; i < flattenDocs.length; i += 1) {
-            finalDocs.push(new Document(flattenDocs[i]))
+            if (flattenDocs[i] && flattenDocs[i].pageContent) {
+                finalDocs.push(new Document(flattenDocs[i]))
+            }
         }
 
         let vectorStore: SingleStoreVectorStore

packages/components/nodes/vectorstores/Vectara/Vectara_Upsert.ts

@@ -132,7 +132,9 @@ class VectaraUpsert_VectorStores implements INode {
         const flattenDocs = docs && docs.length ? flatten(docs) : []
         const finalDocs = []
         for (let i = 0; i < flattenDocs.length; i += 1) {
-            finalDocs.push(new Document(flattenDocs[i]))
+            if (flattenDocs[i] && flattenDocs[i].pageContent) {
+                finalDocs.push(new Document(flattenDocs[i]))
+            }
         }
 
         const vectorStore = await VectaraStore.fromDocuments(finalDocs, embeddings, vectaraArgs)

packages/components/nodes/vectorstores/Weaviate_Upsert/Weaviate_Upsert.ts

@@ -143,7 +143,9 @@ class WeaviateUpsert_VectorStores implements INode {
         const flattenDocs = docs && docs.length ? flatten(docs) : []
         const finalDocs = []
         for (let i = 0; i < flattenDocs.length; i += 1) {
-            finalDocs.push(new Document(flattenDocs[i]))
+            if (flattenDocs[i] && flattenDocs[i].pageContent) {
+                finalDocs.push(new Document(flattenDocs[i]))
+            }
         }
 
         const obj: WeaviateLibArgs = {

packages/components/nodes/vectorstores/Zep/Zep_Upsert.ts

@@ -106,7 +106,9 @@ class Zep_Upsert_VectorStores implements INode {
         const flattenDocs = docs && docs.length ? flatten(docs) : []
         const finalDocs = []
         for (let i = 0; i < flattenDocs.length; i += 1) {
-            finalDocs.push(new Document(flattenDocs[i]))
+            if (flattenDocs[i] && flattenDocs[i].pageContent) {
+                finalDocs.push(new Document(flattenDocs[i]))
+            }
         }
 
         const zepConfig: IZepConfig = {

packages/components/src/utils.ts

@@ -301,7 +301,7 @@ async function crawl(baseURL: string, currentURL: string, pages: string[], limit
 }
 
 /**
- * Prep URL before passing into recursive carwl function
+ * Prep URL before passing into recursive crawl function
  * @param {string} stringURL
  * @param {number} limit
  * @returns {Promise<string[]>}
@@ -445,7 +445,7 @@ export const getCredentialData = async (selectedCredentialId: string, options: I
 
         if (!credential) return {}
 
-        // Decrpyt credentialData
+        // Decrypt credentialData
         const decryptedCredentialData = await decryptCredentialData(credential.encryptedData)
 
         return decryptedCredentialData

packages/server/marketplaces/chatflows/Antonym.json

@@ -45,7 +45,7 @@
                         "id": "fewShotPromptTemplate_1-input-suffix-string"
                     },
                     {
-                        "label": "Example Seperator",
+                        "label": "Example Separator",
                         "name": "exampleSeparator",
                         "type": "string",
                         "placeholder": "\n\n",

packages/server/marketplaces/chatflows/Chat with a Podcast.json

@@ -0,0 +1,669 @@
+{
+    "description": "Engage with data sources such as YouTube Transcripts, Google, and more through intelligent Q&A interactions",
+    "nodes": [
+        {
+            "width": 300,
+            "height": 483,
+            "id": "conversationalRetrievalQAChain_0",
+            "position": {
+                "x": 1499.2693059023254,
+                "y": 430.03911199833317
+            },
+            "type": "customNode",
+            "data": {
+                "id": "conversationalRetrievalQAChain_0",
+                "label": "Conversational Retrieval QA Chain",
+                "version": 1,
+                "name": "conversationalRetrievalQAChain",
+                "type": "ConversationalRetrievalQAChain",
+                "baseClasses": ["ConversationalRetrievalQAChain", "BaseChain", "Runnable"],
+                "category": "Chains",
+                "description": "Document QA - built on RetrievalQAChain to provide a chat history component",
+                "inputParams": [
+                    {
+                        "label": "Return Source Documents",
+                        "name": "returnSourceDocuments",
+                        "type": "boolean",
+                        "optional": true,
+                        "id": "conversationalRetrievalQAChain_0-input-returnSourceDocuments-boolean"
+                    },
+                    {
+                        "label": "System Message",
+                        "name": "systemMessagePrompt",
+                        "type": "string",
+                        "rows": 4,
+                        "additionalParams": true,
+                        "optional": true,
+                        "placeholder": "I want you to act as a document that I am having a conversation with. Your name is \"AI Assistant\". You will provide me with answers from the given info. If the answer is not included, say exactly \"Hmm, I am not sure.\" and stop after that. Refuse to answer any question not about the info. Never break character.",
+                        "id": "conversationalRetrievalQAChain_0-input-systemMessagePrompt-string"
+                    },
+                    {
+                        "label": "Chain Option",
+                        "name": "chainOption",
+                        "type": "options",
+                        "options": [
+                            {
+                                "label": "MapReduceDocumentsChain",
+                                "name": "map_reduce",
+                                "description": "Suitable for QA tasks over larger documents and can run the preprocessing step in parallel, reducing the running time"
+                            },
+                            {
+                                "label": "RefineDocumentsChain",
+                                "name": "refine",
+                                "description": "Suitable for QA tasks over a large number of documents."
+                            },
+                            {
+                                "label": "StuffDocumentsChain",
+                                "name": "stuff",
+                                "description": "Suitable for QA tasks over a small number of documents."
+                            }
+                        ],
+                        "additionalParams": true,
+                        "optional": true,
+                        "id": "conversationalRetrievalQAChain_0-input-chainOption-options"
+                    }
+                ],
+                "inputAnchors": [
+                    {
+                        "label": "Language Model",
+                        "name": "model",
+                        "type": "BaseLanguageModel",
+                        "id": "conversationalRetrievalQAChain_0-input-model-BaseLanguageModel"
+                    },
+                    {
+                        "label": "Vector Store Retriever",
+                        "name": "vectorStoreRetriever",
+                        "type": "BaseRetriever",
+                        "id": "conversationalRetrievalQAChain_0-input-vectorStoreRetriever-BaseRetriever"
+                    },
+                    {
+                        "label": "Memory",
+                        "name": "memory",
+                        "type": "BaseMemory",
+                        "optional": true,
+                        "description": "If left empty, a default BufferMemory will be used",
+                        "id": "conversationalRetrievalQAChain_0-input-memory-BaseMemory"
+                    }
+                ],
+                "inputs": {
+                    "model": "{{chatOpenAI_0.data.instance}}",
+                    "vectorStoreRetriever": "{{memoryVectorStore_0.data.instance}}",
+                    "memory": "",
+                    "returnSourceDocuments": "",
+                    "systemMessagePrompt": "",
+                    "chainOption": ""
+                },
+                "outputAnchors": [
+                    {
+                        "id": "conversationalRetrievalQAChain_0-output-conversationalRetrievalQAChain-ConversationalRetrievalQAChain|BaseChain|Runnable",
+                        "name": "conversationalRetrievalQAChain",
+                        "label": "ConversationalRetrievalQAChain",
+                        "type": "ConversationalRetrievalQAChain | BaseChain | Runnable"
+                    }
+                ],
+                "outputs": {},
+                "selected": false
+            },
+            "selected": false,
+            "positionAbsolute": {
+                "x": 1499.2693059023254,
+                "y": 430.03911199833317
+            },
+            "dragging": false
+        },
+        {
+            "width": 300,
+            "height": 408,
+            "id": "memoryVectorStore_0",
+            "position": {
+                "x": 1082.0280622332507,
+                "y": 589.9990964387842
+            },
+            "type": "customNode",
+            "data": {
+                "id": "memoryVectorStore_0",
+                "label": "In-Memory Vector Store",
+                "version": 1,
+                "name": "memoryVectorStore",
+                "type": "Memory",
+                "baseClasses": ["Memory", "VectorStoreRetriever", "BaseRetriever"],
+                "category": "Vector Stores",
+                "description": "In-memory vectorstore that stores embeddings and does an exact, linear search for the most similar embeddings.",
+                "inputParams": [
+                    {
+                        "label": "Top K",
+                        "name": "topK",
+                        "description": "Number of top results to fetch. Default to 4",
+                        "placeholder": "4",
+                        "type": "number",
+                        "optional": true,
+                        "id": "memoryVectorStore_0-input-topK-number"
+                    }
+                ],
+                "inputAnchors": [
+                    {
+                        "label": "Document",
+                        "name": "document",
+                        "type": "Document",
+                        "list": true,
+                        "id": "memoryVectorStore_0-input-document-Document"
+                    },
+                    {
+                        "label": "Embeddings",
+                        "name": "embeddings",
+                        "type": "Embeddings",
+                        "id": "memoryVectorStore_0-input-embeddings-Embeddings"
+                    }
+                ],
+                "inputs": {
+                    "document": ["{{searchApi_0.data.instance}}", "{{searchApi_0.data.instance}}", "{{searchApi_0.data.instance}}"],
+                    "embeddings": "{{openAIEmbeddings_0.data.instance}}",
+                    "topK": ""
+                },
+                "outputAnchors": [
+                    {
+                        "name": "output",
+                        "label": "Output",
+                        "type": "options",
+                        "options": [
+                            {
+                                "id": "memoryVectorStore_0-output-retriever-Memory|VectorStoreRetriever|BaseRetriever",
+                                "name": "retriever",
+                                "label": "Memory Retriever",
+                                "type": "Memory | VectorStoreRetriever | BaseRetriever"
+                            },
+                            {
+                                "id": "memoryVectorStore_0-output-vectorStore-Memory|VectorStore",
+                                "name": "vectorStore",
+                                "label": "Memory Vector Store",
+                                "type": "Memory | VectorStore"
+                            }
+                        ],
+                        "default": "retriever"
+                    }
+                ],
+                "outputs": {
+                    "output": "retriever"
+                },
+                "selected": false
+            },
+            "positionAbsolute": {
+                "x": 1082.0280622332507,
+                "y": 589.9990964387842
+            },
+            "selected": false,
+            "dragging": false
+        },
+        {
+            "width": 300,
+            "height": 577,
+            "id": "chatOpenAI_0",
+            "position": {
+                "x": 1056.2788608917747,
+                "y": -60.59149112477064
+            },
+            "type": "customNode",
+            "data": {
+                "id": "chatOpenAI_0",
+                "label": "ChatOpenAI",
+                "version": 2,
+                "name": "chatOpenAI",
+                "type": "ChatOpenAI",
+                "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel", "Runnable"],
+                "category": "Chat Models",
+                "description": "Wrapper around OpenAI large language models that use the Chat endpoint",
+                "inputParams": [
+                    {
+                        "label": "Connect Credential",
+                        "name": "credential",
+                        "type": "credential",
+                        "credentialNames": ["openAIApi"],
+                        "id": "chatOpenAI_0-input-credential-credential"
+                    },
+                    {
+                        "label": "Model Name",
+                        "name": "modelName",
+                        "type": "options",
+                        "options": [
+                            {
+                                "label": "gpt-4",
+                                "name": "gpt-4"
+                            },
+                            {
+                                "label": "gpt-4-0613",
+                                "name": "gpt-4-0613"
+                            },
+                            {
+                                "label": "gpt-4-32k",
+                                "name": "gpt-4-32k"
+                            },
+                            {
+                                "label": "gpt-4-32k-0613",
+                                "name": "gpt-4-32k-0613"
+                            },
+                            {
+                                "label": "gpt-3.5-turbo",
+                                "name": "gpt-3.5-turbo"
+                            },
+                            {
+                                "label": "gpt-3.5-turbo-0613",
+                                "name": "gpt-3.5-turbo-0613"
+                            },
+                            {
+                                "label": "gpt-3.5-turbo-16k",
+                                "name": "gpt-3.5-turbo-16k"
+                            },
+                            {
+                                "label": "gpt-3.5-turbo-16k-0613",
+                                "name": "gpt-3.5-turbo-16k-0613"
+                            }
+                        ],
+                        "default": "gpt-3.5-turbo",
+                        "optional": true,
+                        "id": "chatOpenAI_0-input-modelName-options"
+                    },
+                    {
+                        "label": "Temperature",
+                        "name": "temperature",
+                        "type": "number",
+                        "step": 0.1,
+                        "default": 0.9,
+                        "optional": true,
+                        "id": "chatOpenAI_0-input-temperature-number"
+                    },
+                    {
+                        "label": "Max Tokens",
+                        "name": "maxTokens",
+                        "type": "number",
+                        "step": 1,
+                        "optional": true,
+                        "additionalParams": true,
+                        "id": "chatOpenAI_0-input-maxTokens-number"
+                    },
+                    {
+                        "label": "Top Probability",
+                        "name": "topP",
+                        "type": "number",
+                        "step": 0.1,
+                        "optional": true,
+                        "additionalParams": true,
+                        "id": "chatOpenAI_0-input-topP-number"
+                    },
+                    {
+                        "label": "Frequency Penalty",
+                        "name": "frequencyPenalty",
+                        "type": "number",
+                        "step": 0.1,
+                        "optional": true,
+                        "additionalParams": true,
+                        "id": "chatOpenAI_0-input-frequencyPenalty-number"
+                    },
+                    {
+                        "label": "Presence Penalty",
+                        "name": "presencePenalty",
+                        "type": "number",
+                        "step": 0.1,
+                        "optional": true,
+                        "additionalParams": true,
+                        "id": "chatOpenAI_0-input-presencePenalty-number"
+                    },
+                    {
+                        "label": "Timeout",
+                        "name": "timeout",
+                        "type": "number",
+                        "step": 1,
+                        "optional": true,
+                        "additionalParams": true,
+                        "id": "chatOpenAI_0-input-timeout-number"
+                    },
+                    {
+                        "label": "BasePath",
+                        "name": "basepath",
+                        "type": "string",
+                        "optional": true,
+                        "additionalParams": true,
+                        "id": "chatOpenAI_0-input-basepath-string"
+                    },
+                    {
+                        "label": "BaseOptions",
+                        "name": "baseOptions",
+                        "type": "json",
+                        "optional": true,
+                        "additionalParams": true,
+                        "id": "chatOpenAI_0-input-baseOptions-json"
+                    }
+                ],
+                "inputAnchors": [
+                    {
+                        "label": "Cache",
+                        "name": "cache",
+                        "type": "BaseCache",
+                        "optional": true,
+                        "id": "chatOpenAI_0-input-cache-BaseCache"
+                    }
+                ],
+                "inputs": {
+                    "cache": "",
+                    "modelName": "gpt-3.5-turbo",
+                    "temperature": "0.5",
+                    "maxTokens": "",
+                    "topP": "",
+                    "frequencyPenalty": "",
+                    "presencePenalty": "",
+                    "timeout": "",
+                    "basepath": "",
+                    "baseOptions": ""
+                },
+                "outputAnchors": [
+                    {
+                        "id": "chatOpenAI_0-output-chatOpenAI-ChatOpenAI|BaseChatModel|BaseLanguageModel|Runnable",
+                        "name": "chatOpenAI",
+                        "label": "ChatOpenAI",
+                        "type": "ChatOpenAI | BaseChatModel | BaseLanguageModel | Runnable"
+                    }
+                ],
+                "outputs": {},
+                "selected": false
+            },
+            "selected": false,
+            "positionAbsolute": {
+                "x": 1056.2788608917747,
+                "y": -60.59149112477064
+            },
+            "dragging": false
+        },
+        {
+            "width": 300,
+            "height": 478,
+            "id": "characterTextSplitter_0",
+            "position": {
+                "x": 260.5475803279806,
+                "y": -65.1647664861618
+            },
+            "type": "customNode",
+            "data": {
+                "id": "characterTextSplitter_0",
+                "label": "Character Text Splitter",
+                "version": 1,
+                "name": "characterTextSplitter",
+                "type": "CharacterTextSplitter",
+                "baseClasses": ["CharacterTextSplitter", "TextSplitter", "BaseDocumentTransformer", "Runnable"],
+                "category": "Text Splitters",
+                "description": "splits only on one type of character (defaults to \"\\n\\n\").",
+                "inputParams": [
+                    {
+                        "label": "Chunk Size",
+                        "name": "chunkSize",
+                        "type": "number",
+                        "default": 1000,
+                        "optional": true,
+                        "id": "characterTextSplitter_0-input-chunkSize-number"
+                    },
+                    {
+                        "label": "Chunk Overlap",
+                        "name": "chunkOverlap",
+                        "type": "number",
+                        "optional": true,
+                        "id": "characterTextSplitter_0-input-chunkOverlap-number"
+                    },
+                    {
+                        "label": "Custom Separator",
+                        "name": "separator",
+                        "type": "string",
+                        "placeholder": "\" \"",
+                        "description": "Seperator to determine when to split the text, will override the default separator",
+                        "optional": true,
+                        "id": "characterTextSplitter_0-input-separator-string"
+                    }
+                ],
+                "inputAnchors": [],
+                "inputs": {
+                    "chunkSize": "2000",
+                    "chunkOverlap": "200",
+                    "separator": ""
+                },
+                "outputAnchors": [
+                    {
+                        "id": "characterTextSplitter_0-output-characterTextSplitter-CharacterTextSplitter|TextSplitter|BaseDocumentTransformer|Runnable",
+                        "name": "characterTextSplitter",
+                        "label": "CharacterTextSplitter",
+                        "type": "CharacterTextSplitter | TextSplitter | BaseDocumentTransformer | Runnable"
+                    }
+                ],
+                "outputs": {},
+                "selected": false
+            },
+            "selected": false,
+            "positionAbsolute": {
+                "x": 260.5475803279806,
+                "y": -65.1647664861618
+            },
+            "dragging": false
+        },
+        {
+            "width": 300,
+            "height": 332,
+            "id": "openAIEmbeddings_0",
+            "position": {
+                "x": 666.3950526535211,
+                "y": 777.4191705193945
+            },
+            "type": "customNode",
+            "data": {
+                "id": "openAIEmbeddings_0",
+                "label": "OpenAI Embeddings",
+                "version": 1,
+                "name": "openAIEmbeddings",
+                "type": "OpenAIEmbeddings",
+                "baseClasses": ["OpenAIEmbeddings", "Embeddings"],
+                "category": "Embeddings",
+                "description": "OpenAI API to generate embeddings for a given text",
+                "inputParams": [
+                    {
+                        "label": "Connect Credential",
+                        "name": "credential",
+                        "type": "credential",
+                        "credentialNames": ["openAIApi"],
+                        "id": "openAIEmbeddings_0-input-credential-credential"
+                    },
+                    {
+                        "label": "Strip New Lines",
+                        "name": "stripNewLines",
+                        "type": "boolean",
+                        "optional": true,
+                        "additionalParams": true,
+                        "id": "openAIEmbeddings_0-input-stripNewLines-boolean"
+                    },
+                    {
+                        "label": "Batch Size",
+                        "name": "batchSize",
+                        "type": "number",
+                        "optional": true,
+                        "additionalParams": true,
+                        "id": "openAIEmbeddings_0-input-batchSize-number"
+                    },
+                    {
+                        "label": "Timeout",
+                        "name": "timeout",
+                        "type": "number",
+                        "optional": true,
+                        "additionalParams": true,
+                        "id": "openAIEmbeddings_0-input-timeout-number"
+                    },
+                    {
+                        "label": "BasePath",
+                        "name": "basepath",
+                        "type": "string",
+                        "optional": true,
+                        "additionalParams": true,
+                        "id": "openAIEmbeddings_0-input-basepath-string"
+                    }
+                ],
+                "inputAnchors": [],
+                "inputs": {
+                    "stripNewLines": "",
+                    "batchSize": "",
+                    "timeout": "",
+                    "basepath": ""
+                },
+                "outputAnchors": [
+                    {
+                        "id": "openAIEmbeddings_0-output-openAIEmbeddings-OpenAIEmbeddings|Embeddings",
+                        "name": "openAIEmbeddings",
+                        "label": "OpenAIEmbeddings",
+                        "type": "OpenAIEmbeddings | Embeddings"
+                    }
+                ],
+                "outputs": {},
+                "selected": false
+            },
+            "selected": false,
+            "dragging": false,
+            "positionAbsolute": {
+                "x": 666.3950526535211,
+                "y": 777.4191705193945
+            }
+        },
+        {
+            "width": 300,
+            "height": 482,
+            "id": "searchApi_0",
+            "position": {
+                "x": 680.1258121447145,
+                "y": 144.9905217023999
+            },
+            "type": "customNode",
+            "data": {
+                "id": "searchApi_0",
+                "label": "SearchApi",
+                "version": 1,
+                "name": "searchApi",
+                "type": "Document",
+                "baseClasses": ["Document"],
+                "category": "Document Loaders",
+                "description": "Load data from real-time search results",
+                "inputParams": [
+                    {
+                        "label": "Connect Credential",
+                        "name": "credential",
+                        "type": "credential",
+                        "optional": false,
+                        "credentialNames": ["searchApi"],
+                        "id": "searchApi_0-input-credential-credential"
+                    },
+                    {
+                        "label": "Query",
+                        "name": "query",
+                        "type": "string",
+                        "optional": true,
+                        "id": "searchApi_0-input-query-string"
+                    },
+                    {
+                        "label": "Custom Parameters",
+                        "name": "customParameters",
+                        "type": "json",
+                        "optional": true,
+                        "additionalParams": true,
+                        "id": "searchApi_0-input-customParameters-json"
+                    },
+                    {
+                        "label": "Metadata",
+                        "name": "metadata",
+                        "type": "json",
+                        "optional": true,
+                        "additionalParams": true,
+                        "id": "searchApi_0-input-metadata-json"
+                    }
+                ],
+                "inputAnchors": [
+                    {
+                        "label": "Text Splitter",
+                        "name": "textSplitter",
+                        "type": "TextSplitter",
+                        "optional": true,
+                        "id": "searchApi_0-input-textSplitter-TextSplitter"
+                    }
+                ],
+                "inputs": {
+                    "query": "",
+                    "customParameters": "{\"engine\":\"youtube_transcripts\",\"video_id\":\"0e3GPea1Tyg\"}",
+                    "textSplitter": "{{characterTextSplitter_0.data.instance}}",
+                    "metadata": ""
+                },
+                "outputAnchors": [
+                    {
+                        "id": "searchApi_0-output-searchApi-Document",
+                        "name": "searchApi",
+                        "label": "Document",
+                        "type": "Document"
+                    }
+                ],
+                "outputs": {},
+                "selected": false
+            },
+            "selected": false,
+            "positionAbsolute": {
+                "x": 680.1258121447145,
+                "y": 144.9905217023999
+            },
+            "dragging": false
+        }
+    ],
+    "edges": [
+        {
+            "source": "memoryVectorStore_0",
+            "sourceHandle": "memoryVectorStore_0-output-retriever-Memory|VectorStoreRetriever|BaseRetriever",
+            "target": "conversationalRetrievalQAChain_0",
+            "targetHandle": "conversationalRetrievalQAChain_0-input-vectorStoreRetriever-BaseRetriever",
+            "type": "buttonedge",
+            "id": "memoryVectorStore_0-memoryVectorStore_0-output-retriever-Memory|VectorStoreRetriever|BaseRetriever-conversationalRetrievalQAChain_0-conversationalRetrievalQAChain_0-input-vectorStoreRetriever-BaseRetriever",
+            "data": {
+                "label": ""
+            }
+        },
+        {
+            "source": "chatOpenAI_0",
+            "sourceHandle": "chatOpenAI_0-output-chatOpenAI-ChatOpenAI|BaseChatModel|BaseLanguageModel|Runnable",
+            "target": "conversationalRetrievalQAChain_0",
+            "targetHandle": "conversationalRetrievalQAChain_0-input-model-BaseLanguageModel",
+            "type": "buttonedge",
+            "id": "chatOpenAI_0-chatOpenAI_0-output-chatOpenAI-ChatOpenAI|BaseChatModel|BaseLanguageModel|Runnable-conversationalRetrievalQAChain_0-conversationalRetrievalQAChain_0-input-model-BaseLanguageModel",
+            "data": {
+                "label": ""
+            }
+        },
+        {
+            "source": "openAIEmbeddings_0",
+            "sourceHandle": "openAIEmbeddings_0-output-openAIEmbeddings-OpenAIEmbeddings|Embeddings",
+            "target": "memoryVectorStore_0",
+            "targetHandle": "memoryVectorStore_0-input-embeddings-Embeddings",
+            "type": "buttonedge",
+            "id": "openAIEmbeddings_0-openAIEmbeddings_0-output-openAIEmbeddings-OpenAIEmbeddings|Embeddings-memoryVectorStore_0-memoryVectorStore_0-input-embeddings-Embeddings",
+            "data": {
+                "label": ""
+            }
+        },
+        {
+            "source": "characterTextSplitter_0",
+            "sourceHandle": "characterTextSplitter_0-output-characterTextSplitter-CharacterTextSplitter|TextSplitter|BaseDocumentTransformer|Runnable",
+            "target": "searchApi_0",
+            "targetHandle": "searchApi_0-input-textSplitter-TextSplitter",
+            "type": "buttonedge",
+            "id": "characterTextSplitter_0-characterTextSplitter_0-output-characterTextSplitter-CharacterTextSplitter|TextSplitter|BaseDocumentTransformer|Runnable-searchApi_0-searchApi_0-input-textSplitter-TextSplitter",
+            "data": {
+                "label": ""
+            }
+        },
+        {
+            "source": "searchApi_0",
+            "sourceHandle": "searchApi_0-output-searchApi-Document",
+            "target": "memoryVectorStore_0",
+            "targetHandle": "memoryVectorStore_0-input-document-Document",
+            "type": "buttonedge",
+            "id": "searchApi_0-searchApi_0-output-searchApi-Document-memoryVectorStore_0-memoryVectorStore_0-input-document-Document",
+            "data": {
+                "label": ""
+            }
+        }
+    ]
+}

packages/server/marketplaces/chatflows/Conversational Retrieval QA Chain.json

@@ -368,7 +368,7 @@
                 "id": "recursiveCharacterTextSplitter_0",
                 "label": "Recursive Character Text Splitter",
                 "name": "recursiveCharacterTextSplitter",
-                "version": 1,
+                "version": 2,
                 "type": "RecursiveCharacterTextSplitter",
                 "baseClasses": ["RecursiveCharacterTextSplitter", "TextSplitter"],
                 "category": "Text Splitters",
@@ -388,6 +388,17 @@
                         "type": "number",
                         "optional": true,
                         "id": "recursiveCharacterTextSplitter_0-input-chunkOverlap-number"
+                    },
+                    {
+                        "label": "Custom Separators",
+                        "name": "separators",
+                        "type": "string",
+                        "rows": 4,
+                        "description": "Array of custom separators to determine when to split the text, will override the default separators",
+                        "placeholder": "[\"|\", \"##\", \">\", \"-\"]",
+                        "additionalParams": true,
+                        "optional": true,
+                        "id": "recursiveCharacterTextSplitter_0-input-separators-string"
                     }
                 ],
                 "inputAnchors": [],
@@ -426,7 +437,7 @@
                 "id": "textFile_0",
                 "label": "Text File",
                 "name": "textFile",
-                "version": 2,
+                "version": 3,
                 "type": "Document",
                 "baseClasses": ["Document"],
                 "category": "Document Loaders",
@@ -436,7 +447,7 @@
                         "label": "Txt File",
                         "name": "txtFile",
                         "type": "file",
-                        "fileType": ".txt",
+                        "fileType": ".txt, .html, .aspx, .asp, .cpp, .c, .cs, .css, .go, .h, .java, .js, .less, .ts, .php, .proto, .python, .py, .rst, .ruby, .rb, .rs, .scala, .sc, .scss, .sol, .sql, .swift, .markdown, .md, .tex, .ltx, .vb, .xml",
                         "id": "textFile_0-input-txtFile-file"
                     },
                     {

packages/server/marketplaces/chatflows/Local QnA.json

@@ -14,7 +14,7 @@
                 "id": "recursiveCharacterTextSplitter_1",
                 "label": "Recursive Character Text Splitter",
                 "name": "recursiveCharacterTextSplitter",
-                "version": 1,
+                "version": 2,
                 "type": "RecursiveCharacterTextSplitter",
                 "baseClasses": ["RecursiveCharacterTextSplitter", "TextSplitter"],
                 "category": "Text Splitters",
@@ -34,6 +34,17 @@
                         "type": "number",
                         "optional": true,
                         "id": "recursiveCharacterTextSplitter_1-input-chunkOverlap-number"
+                    },
+                    {
+                        "label": "Custom Separators",
+                        "name": "separators",
+                        "type": "string",
+                        "rows": 4,
+                        "description": "Array of custom separators to determine when to split the text, will override the default separators",
+                        "placeholder": "[\"|\", \"##\", \">\", \"-\"]",
+                        "additionalParams": true,
+                        "optional": true,
+                        "id": "recursiveCharacterTextSplitter_1-input-separators-string"
                     }
                 ],
                 "inputAnchors": [],
@@ -373,7 +384,7 @@
                 "id": "textFile_0",
                 "label": "Text File",
                 "name": "textFile",
-                "version": 1,
+                "version": 3,
                 "type": "Document",
                 "baseClasses": ["Document"],
                 "category": "Document Loaders",
@@ -383,7 +394,7 @@
                         "label": "Txt File",
                         "name": "txtFile",
                         "type": "file",
-                        "fileType": ".txt",
+                        "fileType": ".txt, .html, .aspx, .asp, .cpp, .c, .cs, .css, .go, .h, .java, .js, .less, .ts, .php, .proto, .python, .py, .rst, .ruby, .rb, .rs, .scala, .sc, .scss, .sol, .sql, .swift, .markdown, .md, .tex, .ltx, .vb, .xml",
                         "id": "textFile_0-input-txtFile-file"
                     },
                     {

packages/server/marketplaces/chatflows/Metadata Filter Upsert.json

@@ -14,7 +14,7 @@
                 "id": "recursiveCharacterTextSplitter_1",
                 "label": "Recursive Character Text Splitter",
                 "name": "recursiveCharacterTextSplitter",
-                "version": 1,
+                "version": 2,
                 "type": "RecursiveCharacterTextSplitter",
                 "baseClasses": ["RecursiveCharacterTextSplitter", "TextSplitter"],
                 "category": "Text Splitters",
@@ -34,6 +34,17 @@
                         "type": "number",
                         "optional": true,
                         "id": "recursiveCharacterTextSplitter_1-input-chunkOverlap-number"
+                    },
+                    {
+                        "label": "Custom Separators",
+                        "name": "separators",
+                        "type": "string",
+                        "rows": 4,
+                        "description": "Array of custom separators to determine when to split the text, will override the default separators",
+                        "placeholder": "[\"|\", \"##\", \">\", \"-\"]",
+                        "additionalParams": true,
+                        "optional": true,
+                        "id": "recursiveCharacterTextSplitter_1-input-separators-string"
                     }
                 ],
                 "inputAnchors": [],
@@ -72,7 +83,7 @@
                 "id": "textFile_0",
                 "label": "Text File",
                 "name": "textFile",
-                "version": 1,
+                "version": 3,
                 "type": "Document",
                 "baseClasses": ["Document"],
                 "category": "Document Loaders",
@@ -82,7 +93,7 @@
                         "label": "Txt File",
                         "name": "txtFile",
                         "type": "file",
-                        "fileType": ".txt",
+                        "fileType": ".txt, .html, .aspx, .asp, .cpp, .c, .cs, .css, .go, .h, .java, .js, .less, .ts, .php, .proto, .python, .py, .rst, .ruby, .rb, .rs, .scala, .sc, .scss, .sol, .sql, .swift, .markdown, .md, .tex, .ltx, .vb, .xml",
                         "id": "textFile_0-input-txtFile-file"
                     },
                     {

packages/server/marketplaces/chatflows/Prompt Chaining with VectorStore.json

@@ -3,120 +3,11 @@
     "nodes": [
         {
             "width": 300,
-            "height": 503,
+            "height": 329,
-            "id": "pineconeExistingIndex_0",
-            "position": {
-                "x": 1062.7418678410986,
-                "y": -109.27680365777141
-            },
-            "type": "customNode",
-            "data": {
-                "id": "pineconeExistingIndex_0",
-                "label": "Pinecone Load Existing Index",
-                "version": 1,
-                "name": "pineconeExistingIndex",
-                "type": "Pinecone",
-                "baseClasses": ["Pinecone", "VectorStoreRetriever", "BaseRetriever"],
-                "category": "Vector Stores",
-                "description": "Load existing index from Pinecone (i.e: Document has been upserted)",
-                "inputParams": [
-                    {
-                        "label": "Connect Credential",
-                        "name": "credential",
-                        "type": "credential",
-                        "credentialNames": ["pineconeApi"],
-                        "id": "pineconeExistingIndex_0-input-credential-credential"
-                    },
-                    {
-                        "label": "Pinecone Index",
-                        "name": "pineconeIndex",
-                        "type": "string",
-                        "id": "pineconeExistingIndex_0-input-pineconeIndex-string"
-                    },
-                    {
-                        "label": "Pinecone Namespace",
-                        "name": "pineconeNamespace",
-                        "type": "string",
-                        "placeholder": "my-first-namespace",
-                        "additionalParams": true,
-                        "optional": true,
-                        "id": "pineconeExistingIndex_0-input-pineconeNamespace-string"
-                    },
-                    {
-                        "label": "Pinecone Metadata Filter",
-                        "name": "pineconeMetadataFilter",
-                        "type": "json",
-                        "optional": true,
-                        "additionalParams": true,
-                        "id": "pineconeExistingIndex_0-input-pineconeMetadataFilter-json"
-                    },
-                    {
-                        "label": "Top K",
-                        "name": "topK",
-                        "description": "Number of top results to fetch. Default to 4",
-                        "placeholder": "4",
-                        "type": "number",
-                        "additionalParams": true,
-                        "optional": true,
-                        "id": "pineconeExistingIndex_0-input-topK-number"
-                    }
-                ],
-                "inputAnchors": [
-                    {
-                        "label": "Embeddings",
-                        "name": "embeddings",
-                        "type": "Embeddings",
-                        "id": "pineconeExistingIndex_0-input-embeddings-Embeddings"
-                    }
-                ],
-                "inputs": {
-                    "embeddings": "{{openAIEmbeddings_0.data.instance}}",
-                    "pineconeIndex": "newindex",
-                    "pineconeNamespace": "",
-                    "pineconeMetadataFilter": "{}",
-                    "topK": ""
-                },
-                "outputAnchors": [
-                    {
-                        "name": "output",
-                        "label": "Output",
-                        "type": "options",
-                        "options": [
-                            {
-                                "id": "pineconeExistingIndex_0-output-retriever-Pinecone|VectorStoreRetriever|BaseRetriever",
-                                "name": "retriever",
-                                "label": "Pinecone Retriever",
-                                "type": "Pinecone | VectorStoreRetriever | BaseRetriever"
-                            },
-                            {
-                                "id": "pineconeExistingIndex_0-output-vectorStore-Pinecone|VectorStore",
-                                "name": "vectorStore",
-                                "label": "Pinecone Vector Store",
-                                "type": "Pinecone | VectorStore"
-                            }
-                        ],
-                        "default": "retriever"
-                    }
-                ],
-                "outputs": {
-                    "output": "vectorStore"
-                },
-                "selected": false
-            },
-            "selected": false,
-            "positionAbsolute": {
-                "x": 1062.7418678410986,
-                "y": -109.27680365777141
-            },
-            "dragging": false
-        },
-        {
-            "width": 300,
-            "height": 327,
             "id": "openAIEmbeddings_0",
             "position": {
-                "x": 711.3971966563331,
+                "x": 1198.6643452533754,
-                "y": 7.7184225021727
+                "y": -584.4233173804803
             },
             "type": "customNode",
             "data": {
@@ -189,18 +80,18 @@
             },
             "selected": false,
             "positionAbsolute": {
-                "x": 711.3971966563331,
+                "x": 1198.6643452533754,
-                "y": 7.7184225021727
+                "y": -584.4233173804803
             },
             "dragging": false
         },
         {
             "width": 300,
-            "height": 473,
+            "height": 475,
             "id": "promptTemplate_0",
             "position": {
-                "x": 348.2881107399286,
+                "x": 354.2706973608643,
-                "y": -97.74510214137423
+                "y": -122.34815000085804
             },
             "type": "customNode",
             "data": {
@@ -249,18 +140,18 @@
             },
             "selected": false,
             "positionAbsolute": {
-                "x": 348.2881107399286,
+                "x": 354.2706973608643,
-                "y": -97.74510214137423
+                "y": -122.34815000085804
             },
             "dragging": false
         },
         {
             "width": 300,
-            "height": 522,
+            "height": 574,
             "id": "chatOpenAI_0",
             "position": {
-                "x": 335.7621848973805,
+                "x": 353.5672832154869,
-                "y": -721.7411273245009
+                "y": -730.6436764835541
             },
             "type": "customNode",
             "data": {
@@ -396,7 +287,7 @@
                 ],
                 "inputs": {
                     "modelName": "gpt-3.5-turbo-16k",
-                    "temperature": 0.9,
+                    "temperature": "0",
                     "maxTokens": "",
                     "topP": "",
                     "frequencyPenalty": "",
@@ -418,17 +309,17 @@
             "selected": false,
             "dragging": false,
             "positionAbsolute": {
-                "x": 335.7621848973805,
+                "x": 353.5672832154869,
-                "y": -721.7411273245009
+                "y": -730.6436764835541
             }
         },
         {
             "width": 300,
-            "height": 522,
+            "height": 574,
             "id": "chatOpenAI_1",
             "position": {
-                "x": 1765.2801848172305,
+                "x": 2281.9246645710673,
-                "y": -737.9261054149061
+                "y": -778.8379360672121
             },
             "type": "customNode",
             "data": {
@@ -564,7 +455,7 @@
                 ],
                 "inputs": {
                     "modelName": "gpt-3.5-turbo-16k",
-                    "temperature": 0.9,
+                    "temperature": "0",
                     "maxTokens": "",
                     "topP": "",
                     "frequencyPenalty": "",
@@ -586,36 +477,153 @@
             "selected": false,
             "dragging": false,
             "positionAbsolute": {
-                "x": 1765.2801848172305,
+                "x": 2281.9246645710673,
-                "y": -737.9261054149061
+                "y": -778.8379360672121
             }
         },
         {
             "width": 300,
-            "height": 473,
+            "height": 505,
-            "id": "promptTemplate_1",
+            "id": "pineconeExistingIndex_0",
             "position": {
-                "x": 1773.720934090435,
+                "x": 1544.4998097474581,
-                "y": -116.71323227575395
+                "y": -628.8477510577202
             },
             "type": "customNode",
             "data": {
-                "id": "promptTemplate_1",
+                "id": "pineconeExistingIndex_0",
-                "label": "Prompt Template",
+                "label": "Pinecone Load Existing Index",
                 "version": 1,
-                "name": "promptTemplate",
+                "name": "pineconeExistingIndex",
-                "type": "PromptTemplate",
+                "type": "Pinecone",
-                "baseClasses": ["PromptTemplate", "BaseStringPromptTemplate", "BasePromptTemplate", "Runnable"],
+                "baseClasses": ["Pinecone", "VectorStoreRetriever", "BaseRetriever"],
-                "category": "Prompts",
+                "category": "Vector Stores",
-                "description": "Schema to represent a basic prompt for an LLM",
+                "description": "Load existing index from Pinecone (i.e: Document has been upserted)",
                 "inputParams": [
                     {
-                        "label": "Template",
+                        "label": "Connect Credential",
-                        "name": "template",
+                        "name": "credential",
+                        "type": "credential",
+                        "credentialNames": ["pineconeApi"],
+                        "id": "pineconeExistingIndex_0-input-credential-credential"
+                    },
+                    {
+                        "label": "Pinecone Index",
+                        "name": "pineconeIndex",
+                        "type": "string",
+                        "id": "pineconeExistingIndex_0-input-pineconeIndex-string"
+                    },
+                    {
+                        "label": "Pinecone Namespace",
+                        "name": "pineconeNamespace",
+                        "type": "string",
+                        "placeholder": "my-first-namespace",
+                        "additionalParams": true,
+                        "optional": true,
+                        "id": "pineconeExistingIndex_0-input-pineconeNamespace-string"
+                    },
+                    {
+                        "label": "Pinecone Metadata Filter",
+                        "name": "pineconeMetadataFilter",
+                        "type": "json",
+                        "optional": true,
+                        "additionalParams": true,
+                        "id": "pineconeExistingIndex_0-input-pineconeMetadataFilter-json"
+                    },
+                    {
+                        "label": "Top K",
+                        "name": "topK",
+                        "description": "Number of top results to fetch. Default to 4",
+                        "placeholder": "4",
+                        "type": "number",
+                        "additionalParams": true,
+                        "optional": true,
+                        "id": "pineconeExistingIndex_0-input-topK-number"
+                    }
+                ],
+                "inputAnchors": [
+                    {
+                        "label": "Embeddings",
+                        "name": "embeddings",
+                        "type": "Embeddings",
+                        "id": "pineconeExistingIndex_0-input-embeddings-Embeddings"
+                    }
+                ],
+                "inputs": {
+                    "embeddings": "{{openAIEmbeddings_0.data.instance}}",
+                    "pineconeIndex": "",
+                    "pineconeNamespace": "",
+                    "pineconeMetadataFilter": "",
+                    "topK": ""
+                },
+                "outputAnchors": [
+                    {
+                        "name": "output",
+                        "label": "Output",
+                        "type": "options",
+                        "options": [
+                            {
+                                "id": "pineconeExistingIndex_0-output-retriever-Pinecone|VectorStoreRetriever|BaseRetriever",
+                                "name": "retriever",
+                                "label": "Pinecone Retriever",
+                                "type": "Pinecone | VectorStoreRetriever | BaseRetriever"
+                            },
+                            {
+                                "id": "pineconeExistingIndex_0-output-vectorStore-Pinecone|VectorStore",
+                                "name": "vectorStore",
+                                "label": "Pinecone Vector Store",
+                                "type": "Pinecone | VectorStore"
+                            }
+                        ],
+                        "default": "retriever"
+                    }
+                ],
+                "outputs": {
+                    "output": "vectorStore"
+                },
+                "selected": false
+            },
+            "selected": false,
+            "positionAbsolute": {
+                "x": 1544.4998097474581,
+                "y": -628.8477510577202
+            },
+            "dragging": false
+        },
+        {
+            "width": 300,
+            "height": 652,
+            "id": "chatPromptTemplate_0",
+            "position": {
+                "x": 2290.8365353040026,
+                "y": -168.49082887954518
+            },
+            "type": "customNode",
+            "data": {
+                "id": "chatPromptTemplate_0",
+                "label": "Chat Prompt Template",
+                "version": 1,
+                "name": "chatPromptTemplate",
+                "type": "ChatPromptTemplate",
+                "baseClasses": ["ChatPromptTemplate", "BaseChatPromptTemplate", "BasePromptTemplate", "Runnable"],
+                "category": "Prompts",
+                "description": "Schema to represent a chat prompt",
+                "inputParams": [
+                    {
+                        "label": "System Message",
+                        "name": "systemMessagePrompt",
                         "type": "string",
                         "rows": 4,
-                        "placeholder": "What is a good name for a company that makes {product}?",
+                        "placeholder": "You are a helpful assistant that translates {input_language} to {output_language}.",
-                        "id": "promptTemplate_1-input-template-string"
+                        "id": "chatPromptTemplate_0-input-systemMessagePrompt-string"
+                    },
+                    {
+                        "label": "Human Message",
+                        "name": "humanMessagePrompt",
+                        "type": "string",
+                        "rows": 4,
+                        "placeholder": "{text}",
+                        "id": "chatPromptTemplate_0-input-humanMessagePrompt-string"
                     },
                     {
                         "label": "Format Prompt Values",
@@ -624,20 +632,21 @@
                         "optional": true,
                         "acceptVariable": true,
                         "list": true,
-                        "id": "promptTemplate_1-input-promptValues-json"
+                        "id": "chatPromptTemplate_0-input-promptValues-json"
                     }
                 ],
                 "inputAnchors": [],
                 "inputs": {
-                    "template": "Use the following pieces of context to answer the question at the end.\n\n{context}\n\nQuestion: {question}\nHelpful Answer:",
+                    "systemMessagePrompt": "Using the provided context, answer the user's question to the best of your ability using the resources provided. If there is nothing in the context relevant to the question at hand, just say \"Hmm, I'm not sure.\" Don't try to make up an answer.\n\nAnything between the following \\`context\\`  html blocks is retrieved from a knowledge bank, not part of the conversation with the user.\n\n<context>\n    {context}\n<context/>\n\nREMEMBER: If there is no relevant information within the context, just say \"Hmm, I'm not sure.\" Don't try to make up an answer. Anything between the preceding 'context' html blocks is retrieved from a knowledge bank, not part of the conversation with the user.",
-                    "promptValues": "{\"context\":\"{{vectorStoreToDocument_0.data.instance}}\",\"question\":\"{{llmChain_0.data.instance}}\"}"
+                    "humanMessagePrompt": "{text}",
+                    "promptValues": "{\"context\":\"{{vectorStoreToDocument_0.data.instance}}\",\"text\":\"{{question}}\"}"
                 },
                 "outputAnchors": [
                     {
-                        "id": "promptTemplate_1-output-promptTemplate-PromptTemplate|BaseStringPromptTemplate|BasePromptTemplate|Runnable",
+                        "id": "chatPromptTemplate_0-output-chatPromptTemplate-ChatPromptTemplate|BaseChatPromptTemplate|BasePromptTemplate|Runnable",
-                        "name": "promptTemplate",
+                        "name": "chatPromptTemplate",
-                        "label": "PromptTemplate",
+                        "label": "ChatPromptTemplate",
-                        "type": "PromptTemplate | BaseStringPromptTemplate | BasePromptTemplate | Runnable"
+                        "type": "ChatPromptTemplate | BaseChatPromptTemplate | BasePromptTemplate | Runnable"
                     }
                 ],
                 "outputs": {},
@@ -645,18 +654,18 @@
             },
             "selected": false,
             "positionAbsolute": {
-                "x": 1773.720934090435,
+                "x": 2290.8365353040026,
-                "y": -116.71323227575395
+                "y": -168.49082887954518
             },
             "dragging": false
         },
         {
             "width": 300,
-            "height": 404,
+            "height": 405,
             "id": "llmChain_0",
             "position": {
-                "x": 756.1670091985342,
+                "x": 747.1299875516488,
-                "y": -592.5151355056942
+                "y": -267.01184813798244
             },
             "type": "customNode",
             "data": {
@@ -695,7 +704,7 @@
                 "inputs": {
                     "model": "{{chatOpenAI_0.data.instance}}",
                     "prompt": "{{promptTemplate_0.data.instance}}",
-                    "chainName": "QuestionChain"
+                    "chainName": "RephraseQuestion"
                 },
                 "outputAnchors": [
                     {
@@ -726,18 +735,18 @@
             },
             "selected": false,
             "positionAbsolute": {
-                "x": 756.1670091985342,
+                "x": 747.1299875516488,
-                "y": -592.5151355056942
+                "y": -267.01184813798244
             },
             "dragging": false
         },
         {
             "width": 300,
-            "height": 404,
+            "height": 405,
             "id": "llmChain_1",
             "position": {
-                "x": 2200.1274896215496,
+                "x": 2694.8707655351186,
-                "y": -144.29167974642334
+                "y": -308.59150355411236
             },
             "type": "customNode",
             "data": {
@@ -775,8 +784,8 @@
                 ],
                 "inputs": {
                     "model": "{{chatOpenAI_1.data.instance}}",
-                    "prompt": "{{promptTemplate_1.data.instance}}",
+                    "prompt": "{{chatPromptTemplate_0.data.instance}}",
-                    "chainName": ""
+                    "chainName": "FinalResponse"
                 },
                 "outputAnchors": [
                     {
@@ -807,30 +816,39 @@
             },
             "selected": false,
             "positionAbsolute": {
-                "x": 2200.1274896215496,
+                "x": 2694.8707655351186,
-                "y": -144.29167974642334
+                "y": -308.59150355411236
             },
             "dragging": false
         },
         {
             "width": 300,
-            "height": 353,
+            "height": 454,
             "id": "vectorStoreToDocument_0",
             "position": {
-                "x": 1407.7038120189868,
+                "x": 1906.6871314089658,
-                "y": -26.16468811205081
+                "y": -157.0046189166955
             },
             "type": "customNode",
             "data": {
                 "id": "vectorStoreToDocument_0",
                 "label": "VectorStore To Document",
-                "version": 1,
+                "version": 2,
                 "name": "vectorStoreToDocument",
                 "type": "Document",
                 "baseClasses": ["Document"],
                 "category": "Document Loaders",
                 "description": "Search documents with scores from vector store",
                 "inputParams": [
+                    {
+                        "label": "Query",
+                        "name": "query",
+                        "type": "string",
+                        "description": "Query to retrieve documents from vector database. If not specified, user question will be used",
+                        "optional": true,
+                        "acceptVariable": true,
+                        "id": "vectorStoreToDocument_0-input-query-string"
+                    },
                     {
                         "label": "Minimum Score (%)",
                         "name": "minScore",
@@ -852,6 +870,7 @@
                 ],
                 "inputs": {
                     "vectorStore": "{{pineconeExistingIndex_0.data.instance}}",
+                    "query": "{{llmChain_0.data.instance}}",
                     "minScore": ""
                 },
                 "outputAnchors": [
@@ -883,8 +902,8 @@
             },
             "selected": false,
             "positionAbsolute": {
-                "x": 1407.7038120189868,
+                "x": 1906.6871314089658,
-                "y": -26.16468811205081
+                "y": -157.0046189166955
             },
             "dragging": false
         }
@@ -901,6 +920,50 @@
                 "label": ""
             }
         },
+        {
+            "source": "promptTemplate_0",
+            "sourceHandle": "promptTemplate_0-output-promptTemplate-PromptTemplate|BaseStringPromptTemplate|BasePromptTemplate|Runnable",
+            "target": "llmChain_0",
+            "targetHandle": "llmChain_0-input-prompt-BasePromptTemplate",
+            "type": "buttonedge",
+            "id": "promptTemplate_0-promptTemplate_0-output-promptTemplate-PromptTemplate|BaseStringPromptTemplate|BasePromptTemplate|Runnable-llmChain_0-llmChain_0-input-prompt-BasePromptTemplate",
+            "data": {
+                "label": ""
+            }
+        },
+        {
+            "source": "chatOpenAI_0",
+            "sourceHandle": "chatOpenAI_0-output-chatOpenAI-ChatOpenAI|BaseChatModel|BaseLanguageModel|Runnable",
+            "target": "llmChain_0",
+            "targetHandle": "llmChain_0-input-model-BaseLanguageModel",
+            "type": "buttonedge",
+            "id": "chatOpenAI_0-chatOpenAI_0-output-chatOpenAI-ChatOpenAI|BaseChatModel|BaseLanguageModel|Runnable-llmChain_0-llmChain_0-input-model-BaseLanguageModel",
+            "data": {
+                "label": ""
+            }
+        },
+        {
+            "source": "chatPromptTemplate_0",
+            "sourceHandle": "chatPromptTemplate_0-output-chatPromptTemplate-ChatPromptTemplate|BaseChatPromptTemplate|BasePromptTemplate|Runnable",
+            "target": "llmChain_1",
+            "targetHandle": "llmChain_1-input-prompt-BasePromptTemplate",
+            "type": "buttonedge",
+            "id": "chatPromptTemplate_0-chatPromptTemplate_0-output-chatPromptTemplate-ChatPromptTemplate|BaseChatPromptTemplate|BasePromptTemplate|Runnable-llmChain_1-llmChain_1-input-prompt-BasePromptTemplate",
+            "data": {
+                "label": ""
+            }
+        },
+        {
+            "source": "chatOpenAI_1",
+            "sourceHandle": "chatOpenAI_1-output-chatOpenAI-ChatOpenAI|BaseChatModel|BaseLanguageModel|Runnable",
+            "target": "llmChain_1",
+            "targetHandle": "llmChain_1-input-model-BaseLanguageModel",
+            "type": "buttonedge",
+            "id": "chatOpenAI_1-chatOpenAI_1-output-chatOpenAI-ChatOpenAI|BaseChatModel|BaseLanguageModel|Runnable-llmChain_1-llmChain_1-input-model-BaseLanguageModel",
+            "data": {
+                "label": ""
+            }
+        },
         {
             "source": "pineconeExistingIndex_0",
             "sourceHandle": "pineconeExistingIndex_0-output-vectorStore-Pinecone|VectorStore",
@@ -915,32 +978,10 @@
         {
             "source": "vectorStoreToDocument_0",
             "sourceHandle": "vectorStoreToDocument_0-output-text-string|json",
-            "target": "promptTemplate_1",
+            "target": "chatPromptTemplate_0",
-            "targetHandle": "promptTemplate_1-input-promptValues-json",
+            "targetHandle": "chatPromptTemplate_0-input-promptValues-json",
             "type": "buttonedge",
-            "id": "vectorStoreToDocument_0-vectorStoreToDocument_0-output-text-string|json-promptTemplate_1-promptTemplate_1-input-promptValues-json",
+            "id": "vectorStoreToDocument_0-vectorStoreToDocument_0-output-text-string|json-chatPromptTemplate_0-chatPromptTemplate_0-input-promptValues-json",
-            "data": {
-                "label": ""
-            }
-        },
-        {
-            "source": "chatOpenAI_0",
-            "sourceHandle": "chatOpenAI_0-output-chatOpenAI-ChatOpenAI|BaseChatModel|BaseLanguageModel|Runnable",
-            "target": "llmChain_0",
-            "targetHandle": "llmChain_0-input-model-BaseLanguageModel",
-            "type": "buttonedge",
-            "id": "chatOpenAI_0-chatOpenAI_0-output-chatOpenAI-ChatOpenAI|BaseChatModel|BaseLanguageModel|Runnable-llmChain_0-llmChain_0-input-model-BaseLanguageModel",
-            "data": {
-                "label": ""
-            }
-        },
-        {
-            "source": "promptTemplate_0",
-            "sourceHandle": "promptTemplate_0-output-promptTemplate-PromptTemplate|BaseStringPromptTemplate|BasePromptTemplate|Runnable",
-            "target": "llmChain_0",
-            "targetHandle": "llmChain_0-input-prompt-BasePromptTemplate",
-            "type": "buttonedge",
-            "id": "promptTemplate_0-promptTemplate_0-output-promptTemplate-PromptTemplate|BaseStringPromptTemplate|BasePromptTemplate|Runnable-llmChain_0-llmChain_0-input-prompt-BasePromptTemplate",
             "data": {
                 "label": ""
             }
@@ -948,32 +989,10 @@
         {
             "source": "llmChain_0",
             "sourceHandle": "llmChain_0-output-outputPrediction-string|json",
-            "target": "promptTemplate_1",
+            "target": "vectorStoreToDocument_0",
-            "targetHandle": "promptTemplate_1-input-promptValues-json",
+            "targetHandle": "vectorStoreToDocument_0-input-query-string",
             "type": "buttonedge",
-            "id": "llmChain_0-llmChain_0-output-outputPrediction-string|json-promptTemplate_1-promptTemplate_1-input-promptValues-json",
+            "id": "llmChain_0-llmChain_0-output-outputPrediction-string|json-vectorStoreToDocument_0-vectorStoreToDocument_0-input-query-string",
-            "data": {
-                "label": ""
-            }
-        },
-        {
-            "source": "chatOpenAI_1",
-            "sourceHandle": "chatOpenAI_1-output-chatOpenAI-ChatOpenAI|BaseChatModel|BaseLanguageModel|Runnable",
-            "target": "llmChain_1",
-            "targetHandle": "llmChain_1-input-model-BaseLanguageModel",
-            "type": "buttonedge",
-            "id": "chatOpenAI_1-chatOpenAI_1-output-chatOpenAI-ChatOpenAI|BaseChatModel|BaseLanguageModel|Runnable-llmChain_1-llmChain_1-input-model-BaseLanguageModel",
-            "data": {
-                "label": ""
-            }
-        },
-        {
-            "source": "promptTemplate_1",
-            "sourceHandle": "promptTemplate_1-output-promptTemplate-PromptTemplate|BaseStringPromptTemplate|BasePromptTemplate|Runnable",
-            "target": "llmChain_1",
-            "targetHandle": "llmChain_1-input-prompt-BasePromptTemplate",
-            "type": "buttonedge",
-            "id": "promptTemplate_1-promptTemplate_1-output-promptTemplate-PromptTemplate|BaseStringPromptTemplate|BasePromptTemplate|Runnable-llmChain_1-llmChain_1-input-prompt-BasePromptTemplate",
             "data": {
                 "label": ""
             }

packages/ui/src/ui-component/input/Input.js

@@ -1,10 +1,39 @@
-import { useState } from 'react'
+import { useState, useEffect, useRef } from 'react'
 import PropTypes from 'prop-types'
-import { FormControl, OutlinedInput } from '@mui/material'
+import { FormControl, OutlinedInput, Popover } from '@mui/material'
 import ExpandTextDialog from 'ui-component/dialog/ExpandTextDialog'
+import SelectVariable from 'ui-component/json/SelectVariable'
+import { getAvailableNodesForVariable } from 'utils/genericHelper'
 
-export const Input = ({ inputParam, value, onChange, disabled = false, showDialog, dialogProps, onDialogCancel, onDialogConfirm }) => {
+export const Input = ({
+    inputParam,
+    value,
+    nodes,
+    edges,
+    nodeId,
+    onChange,
+    disabled = false,
+    showDialog,
+    dialogProps,
+    onDialogCancel,
+    onDialogConfirm
+}) => {
     const [myValue, setMyValue] = useState(value ?? '')
+    const [anchorEl, setAnchorEl] = useState(null)
+    const [availableNodesForVariable, setAvailableNodesForVariable] = useState([])
+    const ref = useRef(null)
+
+    const openPopOver = Boolean(anchorEl)
+
+    const handleClosePopOver = () => {
+        setAnchorEl(null)
+    }
+
+    const setNewVal = (val) => {
+        const newVal = myValue + val.substring(2)
+        onChange(newVal)
+        setMyValue(newVal)
+    }
 
     const getInputType = (type) => {
         switch (type) {
@@ -19,6 +48,19 @@ export const Input = ({ inputParam, value, onChange, disabled = false, showDialo
         }
     }
 
+    useEffect(() => {
+        if (!disabled && nodes && edges && nodeId && inputParam) {
+            const nodesForVariable = inputParam?.acceptVariable ? getAvailableNodesForVariable(nodes, edges, nodeId, inputParam.id) : []
+            setAvailableNodesForVariable(nodesForVariable)
+        }
+    }, [disabled, inputParam, nodes, edges, nodeId])
+
+    useEffect(() => {
+        if (typeof myValue === 'string' && myValue && myValue.endsWith('{{')) {
+            setAnchorEl(ref.current)
+        }
+    }, [myValue])
+
     return (
         <>
             <FormControl sx={{ mt: 1, width: '100%' }} size='small'>
@@ -55,6 +97,31 @@ export const Input = ({ inputParam, value, onChange, disabled = false, showDialo
                     }}
                 ></ExpandTextDialog>
             )}
+            <div ref={ref}></div>
+            {inputParam?.acceptVariable && (
+                <Popover
+                    open={openPopOver}
+                    anchorEl={anchorEl}
+                    onClose={handleClosePopOver}
+                    anchorOrigin={{
+                        vertical: 'bottom',
+                        horizontal: 'left'
+                    }}
+                    transformOrigin={{
+                        vertical: 'top',
+                        horizontal: 'left'
+                    }}
+                >
+                    <SelectVariable
+                        disabled={disabled}
+                        availableNodesForVariable={availableNodesForVariable}
+                        onSelectAndReturnVal={(val) => {
+                            setNewVal(val)
+                            handleClosePopOver()
+                        }}
+                    />
+                </Popover>
+            )}
         </>
     )
 }
@@ -66,6 +133,9 @@ Input.propTypes = {
     disabled: PropTypes.bool,
     showDialog: PropTypes.bool,
     dialogProps: PropTypes.object,
+    nodes: PropTypes.array,
+    edges: PropTypes.array,
+    nodeId: PropTypes.string,
     onDialogCancel: PropTypes.func,
     onDialogConfirm: PropTypes.func
 }

packages/ui/src/views/canvas/NodeInputHandler.js

@@ -265,6 +265,9 @@ const NodeInputHandler = ({ inputAnchor, inputParam, data, disabled = false, isA
                                 inputParam={inputParam}
                                 onChange={(newValue) => (data.inputs[inputParam.name] = newValue)}
                                 value={data.inputs[inputParam.name] ?? inputParam.default ?? ''}
+                                nodes={inputParam?.acceptVariable && reactFlowInstance ? reactFlowInstance.getNodes() : []}
+                                edges={inputParam?.acceptVariable && reactFlowInstance ? reactFlowInstance.getEdges() : []}
+                                nodeId={data.id}
                                 showDialog={showExpandDialog}
                                 dialogProps={expandDialogProps}
                                 onDialogCancel={() => setShowExpandDialog(false)}