Merge branch 'main' into feature/Credential

# Conflicts:
#	README.md
#	docker/.env.example
#	packages/components/nodes/documentloaders/Notion/NotionDB.ts
#	packages/components/nodes/memory/DynamoDb/DynamoDb.ts
#	packages/components/nodes/memory/MotorheadMemory/MotorheadMemory.ts
#	packages/components/nodes/memory/ZepMemory/ZepMemory.ts
#	packages/components/package.json
#	packages/components/src/utils.ts
#	packages/server/.env.example
#	packages/server/README.md
#	packages/server/marketplaces/chatflows/Conversational Retrieval QA Chain.json
#	packages/server/src/ChildProcess.ts
#	packages/server/src/DataSource.ts
#	packages/server/src/commands/start.ts
#	packages/server/src/index.ts
#	packages/server/src/utils/index.ts
#	packages/server/src/utils/logger.ts

packages/components/nodes/agents/AirtableAgent/AirtableAgent.ts

@@ -0,0 +1,231 @@
+import { ICommonObject, INode, INodeData, INodeParams, PromptTemplate } from '../../../src/Interface'
+import { AgentExecutor } from 'langchain/agents'
+import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
+import { LoadPyodide, finalSystemPrompt, systemPrompt } from './core'
+import { LLMChain } from 'langchain/chains'
+import { BaseLanguageModel } from 'langchain/base_language'
+import { ConsoleCallbackHandler, CustomChainHandler } from '../../../src/handler'
+import axios from 'axios'
+
+class Airtable_Agents implements INode {
+    label: string
+    name: string
+    description: string
+    type: string
+    icon: string
+    category: string
+    baseClasses: string[]
+    credential: INodeParams
+    inputs: INodeParams[]
+
+    constructor() {
+        this.label = 'Airtable Agent'
+        this.name = 'airtableAgent'
+        this.type = 'AgentExecutor'
+        this.category = 'Agents'
+        this.icon = 'airtable.svg'
+        this.description = 'Agent used to to answer queries on Airtable table'
+        this.baseClasses = [this.type, ...getBaseClasses(AgentExecutor)]
+        this.credential = {
+            label: 'Connect Credential',
+            name: 'credential',
+            type: 'credential',
+            credentialNames: ['airtableApi']
+        }
+        this.inputs = [
+            {
+                label: 'Language Model',
+                name: 'model',
+                type: 'BaseLanguageModel'
+            },
+            {
+                label: 'Base Id',
+                name: 'baseId',
+                type: 'string',
+                placeholder: 'app11RobdGoX0YNsC',
+                description:
+                    'If your table URL looks like: https://airtable.com/app11RobdGoX0YNsC/tblJdmvbrgizbYICO/viw9UrP77Id0CE4ee, app11RovdGoX0YNsC is the base id'
+            },
+            {
+                label: 'Table Id',
+                name: 'tableId',
+                type: 'string',
+                placeholder: 'tblJdmvbrgizbYICO',
+                description:
+                    'If your table URL looks like: https://airtable.com/app11RobdGoX0YNsC/tblJdmvbrgizbYICO/viw9UrP77Id0CE4ee, tblJdmvbrgizbYICO is the table id'
+            },
+            {
+                label: 'Return All',
+                name: 'returnAll',
+                type: 'boolean',
+                default: true,
+                additionalParams: true,
+                description: 'If all results should be returned or only up to a given limit'
+            },
+            {
+                label: 'Limit',
+                name: 'limit',
+                type: 'number',
+                default: 100,
+                step: 1,
+                additionalParams: true,
+                description: 'Number of results to return'
+            }
+        ]
+    }
+
+    async init(): Promise<any> {
+        // Not used
+        return undefined
+    }
+
+    async run(nodeData: INodeData, input: string, options: ICommonObject): Promise<string> {
+        const model = nodeData.inputs?.model as BaseLanguageModel
+        const baseId = nodeData.inputs?.baseId as string
+        const tableId = nodeData.inputs?.tableId as string
+        const returnAll = nodeData.inputs?.returnAll as boolean
+        const limit = nodeData.inputs?.limit as string
+
+        const credentialData = await getCredentialData(nodeData.credential ?? '', options)
+        const accessToken = getCredentialParam('accessToken', credentialData, nodeData)
+
+        let airtableData: ICommonObject[] = []
+
+        if (returnAll) {
+            airtableData = await loadAll(baseId, tableId, accessToken)
+        } else {
+            airtableData = await loadLimit(limit ? parseInt(limit, 10) : 100, baseId, tableId, accessToken)
+        }
+
+        let base64String = Buffer.from(JSON.stringify(airtableData)).toString('base64')
+
+        const loggerHandler = new ConsoleCallbackHandler(options.logger)
+        const handler = new CustomChainHandler(options.socketIO, options.socketIOClientId)
+
+        const pyodide = await LoadPyodide()
+
+        // First load the csv file and get the dataframe dictionary of column types
+        // For example using titanic.csv: {'PassengerId': 'int64', 'Survived': 'int64', 'Pclass': 'int64', 'Name': 'object', 'Sex': 'object', 'Age': 'float64', 'SibSp': 'int64', 'Parch': 'int64', 'Ticket': 'object', 'Fare': 'float64', 'Cabin': 'object', 'Embarked': 'object'}
+        let dataframeColDict = ''
+        try {
+            const code = `import pandas as pd
+import base64
+import json
+
+base64_string = "${base64String}"
+
+decoded_data = base64.b64decode(base64_string)
+
+json_data = json.loads(decoded_data)
+
+df = pd.DataFrame(json_data)
+my_dict = df.dtypes.astype(str).to_dict()
+print(my_dict)
+json.dumps(my_dict)`
+            dataframeColDict = await pyodide.runPythonAsync(code)
+        } catch (error) {
+            throw new Error(error)
+        }
+
+        // Then tell GPT to come out with ONLY python code
+        // For example: len(df), df[df['SibSp'] > 3]['PassengerId'].count()
+        let pythonCode = ''
+        if (dataframeColDict) {
+            const chain = new LLMChain({
+                llm: model,
+                prompt: PromptTemplate.fromTemplate(systemPrompt),
+                verbose: process.env.DEBUG === 'true' ? true : false
+            })
+            const inputs = {
+                dict: dataframeColDict,
+                question: input
+            }
+            const res = await chain.call(inputs, [loggerHandler])
+            pythonCode = res?.text
+        }
+
+        // Then run the code using Pyodide
+        let finalResult = ''
+        if (pythonCode) {
+            try {
+                const code = `import pandas as pd\n${pythonCode}`
+                finalResult = await pyodide.runPythonAsync(code)
+            } catch (error) {
+                throw new Error(`Sorry, I'm unable to find answer for question: "${input}" using follwoing code: "${pythonCode}"`)
+            }
+        }
+
+        // Finally, return a complete answer
+        if (finalResult) {
+            const chain = new LLMChain({
+                llm: model,
+                prompt: PromptTemplate.fromTemplate(finalSystemPrompt),
+                verbose: process.env.DEBUG === 'true' ? true : false
+            })
+            const inputs = {
+                question: input,
+                answer: finalResult
+            }
+
+            if (options.socketIO && options.socketIOClientId) {
+                const result = await chain.call(inputs, [loggerHandler, handler])
+                return result?.text
+            } else {
+                const result = await chain.call(inputs, [loggerHandler])
+                return result?.text
+            }
+        }
+
+        return pythonCode
+    }
+}
+
+interface AirtableLoaderResponse {
+    records: AirtableLoaderPage[]
+    offset?: string
+}
+
+interface AirtableLoaderPage {
+    id: string
+    createdTime: string
+    fields: ICommonObject
+}
+
+const fetchAirtableData = async (url: string, params: ICommonObject, accessToken: string): Promise<AirtableLoaderResponse> => {
+    try {
+        const headers = {
+            Authorization: `Bearer ${accessToken}`,
+            'Content-Type': 'application/json',
+            Accept: 'application/json'
+        }
+        const response = await axios.get(url, { params, headers })
+        return response.data
+    } catch (error) {
+        throw new Error(`Failed to fetch ${url} from Airtable: ${error}`)
+    }
+}
+
+const loadAll = async (baseId: string, tableId: string, accessToken: string): Promise<ICommonObject[]> => {
+    const params: ICommonObject = { pageSize: 100 }
+    let data: AirtableLoaderResponse
+    let returnPages: AirtableLoaderPage[] = []
+
+    do {
+        data = await fetchAirtableData(`https://api.airtable.com/v0/${baseId}/${tableId}`, params, accessToken)
+        returnPages.push.apply(returnPages, data.records)
+        params.offset = data.offset
+    } while (data.offset !== undefined)
+
+    return data.records.map((page) => page.fields)
+}
+
+const loadLimit = async (limit: number, baseId: string, tableId: string, accessToken: string): Promise<ICommonObject[]> => {
+    const params = { maxRecords: limit }
+    const data = await fetchAirtableData(`https://api.airtable.com/v0/${baseId}/${tableId}`, params, accessToken)
+    if (data.records.length === 0) {
+        return []
+    }
+    return data.records.map((page) => page.fields)
+}
+
+module.exports = { nodeClass: Airtable_Agents }

packages/components/nodes/agents/AirtableAgent/airtable.svg

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<svg width="256px" height="215px" viewBox="0 0 256 215" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" preserveAspectRatio="xMidYMid">
+	<g>
+		<path d="M114.25873,2.70101695 L18.8604023,42.1756384 C13.5552723,44.3711638 13.6102328,51.9065311 18.9486282,54.0225085 L114.746142,92.0117514 C123.163769,95.3498757 132.537419,95.3498757 140.9536,92.0117514 L236.75256,54.0225085 C242.08951,51.9065311 242.145916,44.3711638 236.83934,42.1756384 L141.442459,2.70101695 C132.738459,-0.900338983 122.961284,-0.900338983 114.25873,2.70101695" fill="#FFBF00"></path>
+		<path d="M136.349071,112.756863 L136.349071,207.659101 C136.349071,212.173089 140.900664,215.263892 145.096461,213.600615 L251.844122,172.166219 C254.281184,171.200072 255.879376,168.845451 255.879376,166.224705 L255.879376,71.3224678 C255.879376,66.8084791 251.327783,63.7176768 247.131986,65.3809537 L140.384325,106.815349 C137.94871,107.781496 136.349071,110.136118 136.349071,112.756863" fill="#26B5F8"></path>
+		<path d="M111.422771,117.65355 L79.742409,132.949912 L76.5257763,134.504714 L9.65047684,166.548104 C5.4112904,168.593211 0.000578531073,165.503855 0.000578531073,160.794612 L0.000578531073,71.7210757 C0.000578531073,70.0173017 0.874160452,68.5463864 2.04568588,67.4384994 C2.53454463,66.9481944 3.08848814,66.5446689 3.66412655,66.2250305 C5.26231864,65.2661153 7.54173107,65.0101153 9.47981017,65.7766689 L110.890522,105.957098 C116.045234,108.002206 116.450206,115.225166 111.422771,117.65355" fill="#ED3049"></path>
+		<path d="M111.422771,117.65355 L79.742409,132.949912 L2.04568588,67.4384994 C2.53454463,66.9481944 3.08848814,66.5446689 3.66412655,66.2250305 C5.26231864,65.2661153 7.54173107,65.0101153 9.47981017,65.7766689 L110.890522,105.957098 C116.045234,108.002206 116.450206,115.225166 111.422771,117.65355" fill-opacity="0.25" fill="#000000"></path>
+	</g>
+</svg>

packages/components/nodes/agents/AirtableAgent/core.ts

@@ -0,0 +1,29 @@
+import type { PyodideInterface } from 'pyodide'
+import * as path from 'path'
+import { getUserHome } from '../../../src/utils'
+
+let pyodideInstance: PyodideInterface | undefined
+
+export async function LoadPyodide(): Promise<PyodideInterface> {
+    if (pyodideInstance === undefined) {
+        const { loadPyodide } = await import('pyodide')
+        const obj: any = { packageCacheDir: path.join(getUserHome(), '.flowise', 'pyodideCacheDir') }
+        pyodideInstance = await loadPyodide(obj)
+        await pyodideInstance.loadPackage(['pandas', 'numpy'])
+    }
+
+    return pyodideInstance
+}
+
+export const systemPrompt = `You are working with a pandas dataframe in Python. The name of the dataframe is df.
+
+The columns and data types of a dataframe are given below as a Python dictionary with keys showing column names and values showing the data types.
+{dict}
+
+I will ask question, and you will output the Python code using pandas dataframe to answer my question. Do not provide any explanations. Do not respond with anything except the output of the code.
+
+Question: {question}
+Output Code:`
+
+export const finalSystemPrompt = `You are given the question: {question}. You have an answer to the question: {answer}. Rephrase the answer into a standalone answer.
+Standalone Answer:`

packages/components/nodes/agents/CSVAgent/CSVAgent.ts

@@ -0,0 +1,149 @@
+import { ICommonObject, INode, INodeData, INodeParams, PromptTemplate } from '../../../src/Interface'
+import { AgentExecutor } from 'langchain/agents'
+import { getBaseClasses } from '../../../src/utils'
+import { LoadPyodide, finalSystemPrompt, systemPrompt } from './core'
+import { LLMChain } from 'langchain/chains'
+import { BaseLanguageModel } from 'langchain/base_language'
+import { ConsoleCallbackHandler, CustomChainHandler } from '../../../src/handler'
+
+class CSV_Agents implements INode {
+    label: string
+    name: string
+    description: string
+    type: string
+    icon: string
+    category: string
+    baseClasses: string[]
+    inputs: INodeParams[]
+
+    constructor() {
+        this.label = 'CSV Agent'
+        this.name = 'csvAgent'
+        this.type = 'AgentExecutor'
+        this.category = 'Agents'
+        this.icon = 'csvagent.png'
+        this.description = 'Agent used to to answer queries on CSV data'
+        this.baseClasses = [this.type, ...getBaseClasses(AgentExecutor)]
+        this.inputs = [
+            {
+                label: 'Csv File',
+                name: 'csvFile',
+                type: 'file',
+                fileType: '.csv'
+            },
+            {
+                label: 'Language Model',
+                name: 'model',
+                type: 'BaseLanguageModel'
+            }
+        ]
+    }
+
+    async init(): Promise<any> {
+        // Not used
+        return undefined
+    }
+
+    async run(nodeData: INodeData, input: string, options: ICommonObject): Promise<string> {
+        const csvFileBase64 = nodeData.inputs?.csvFile as string
+        const model = nodeData.inputs?.model as BaseLanguageModel
+
+        const loggerHandler = new ConsoleCallbackHandler(options.logger)
+        const handler = new CustomChainHandler(options.socketIO, options.socketIOClientId)
+
+        let files: string[] = []
+
+        if (csvFileBase64.startsWith('[') && csvFileBase64.endsWith(']')) {
+            files = JSON.parse(csvFileBase64)
+        } else {
+            files = [csvFileBase64]
+        }
+
+        let base64String = ''
+
+        for (const file of files) {
+            const splitDataURI = file.split(',')
+            splitDataURI.pop()
+            base64String = splitDataURI.pop() ?? ''
+        }
+
+        const pyodide = await LoadPyodide()
+
+        // First load the csv file and get the dataframe dictionary of column types
+        // For example using titanic.csv: {'PassengerId': 'int64', 'Survived': 'int64', 'Pclass': 'int64', 'Name': 'object', 'Sex': 'object', 'Age': 'float64', 'SibSp': 'int64', 'Parch': 'int64', 'Ticket': 'object', 'Fare': 'float64', 'Cabin': 'object', 'Embarked': 'object'}
+        let dataframeColDict = ''
+        try {
+            const code = `import pandas as pd
+import base64
+from io import StringIO
+import json
+
+base64_string = "${base64String}"
+
+decoded_data = base64.b64decode(base64_string)
+
+csv_data = StringIO(decoded_data.decode('utf-8'))
+
+df = pd.read_csv(csv_data)
+my_dict = df.dtypes.astype(str).to_dict()
+print(my_dict)
+json.dumps(my_dict)`
+            dataframeColDict = await pyodide.runPythonAsync(code)
+        } catch (error) {
+            throw new Error(error)
+        }
+
+        // Then tell GPT to come out with ONLY python code
+        // For example: len(df), df[df['SibSp'] > 3]['PassengerId'].count()
+        let pythonCode = ''
+        if (dataframeColDict) {
+            const chain = new LLMChain({
+                llm: model,
+                prompt: PromptTemplate.fromTemplate(systemPrompt),
+                verbose: process.env.DEBUG === 'true' ? true : false
+            })
+            const inputs = {
+                dict: dataframeColDict,
+                question: input
+            }
+            const res = await chain.call(inputs, [loggerHandler])
+            pythonCode = res?.text
+        }
+
+        // Then run the code using Pyodide
+        let finalResult = ''
+        if (pythonCode) {
+            try {
+                const code = `import pandas as pd\n${pythonCode}`
+                finalResult = await pyodide.runPythonAsync(code)
+            } catch (error) {
+                throw new Error(`Sorry, I'm unable to find answer for question: "${input}" using follwoing code: "${pythonCode}"`)
+            }
+        }
+
+        // Finally, return a complete answer
+        if (finalResult) {
+            const chain = new LLMChain({
+                llm: model,
+                prompt: PromptTemplate.fromTemplate(finalSystemPrompt),
+                verbose: process.env.DEBUG === 'true' ? true : false
+            })
+            const inputs = {
+                question: input,
+                answer: finalResult
+            }
+
+            if (options.socketIO && options.socketIOClientId) {
+                const result = await chain.call(inputs, [loggerHandler, handler])
+                return result?.text
+            } else {
+                const result = await chain.call(inputs, [loggerHandler])
+                return result?.text
+            }
+        }
+
+        return pythonCode
+    }
+}
+
+module.exports = { nodeClass: CSV_Agents }

packages/components/nodes/agents/CSVAgent/core.ts

@@ -0,0 +1,29 @@
+import type { PyodideInterface } from 'pyodide'
+import * as path from 'path'
+import { getUserHome } from '../../../src/utils'
+
+let pyodideInstance: PyodideInterface | undefined
+
+export async function LoadPyodide(): Promise<PyodideInterface> {
+    if (pyodideInstance === undefined) {
+        const { loadPyodide } = await import('pyodide')
+        const obj: any = { packageCacheDir: path.join(getUserHome(), '.flowise', 'pyodideCacheDir') }
+        pyodideInstance = await loadPyodide(obj)
+        await pyodideInstance.loadPackage(['pandas', 'numpy'])
+    }
+
+    return pyodideInstance
+}
+
+export const systemPrompt = `You are working with a pandas dataframe in Python. The name of the dataframe is df.
+
+The columns and data types of a dataframe are given below as a Python dictionary with keys showing column names and values showing the data types.
+{dict}
+
+I will ask question, and you will output the Python code using pandas dataframe to answer my question. Do not provide any explanations. Do not respond with anything except the output of the code.
+
+Question: {question}
+Output Code:`
+
+export const finalSystemPrompt = `You are given the question: {question}. You have an answer to the question: {answer}. Rephrase the answer into a standalone answer.
+Standalone Answer:`

packages/components/nodes/agents/OpenAIFunctionAgent/OpenAIFunctionAgent.ts

@@ -22,7 +22,7 @@ class OpenAIFunctionAgent_Agents implements INode {
         this.name = 'openAIFunctionAgent'
         this.type = 'AgentExecutor'
         this.category = 'Agents'
-        this.icon = 'openai.svg'
+        this.icon = 'openai.png'
         this.description = `An agent that uses OpenAI's Function Calling functionality to pick the tool and args to call`
         this.baseClasses = [this.type, ...getBaseClasses(AgentExecutor)]
         this.inputs = [

packages/components/nodes/agents/OpenAIFunctionAgent/openai.svg

@@ -1 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0,0,256,256" width="96px" height="96px" fill-rule="nonzero"><g fill="#00a67e" fill-rule="nonzero" stroke="none" stroke-width="1" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="10" stroke-dasharray="" stroke-dashoffset="0" font-family="none" font-weight="none" font-size="none" text-anchor="none" style="mix-blend-mode: normal"><g transform="scale(10.66667,10.66667)"><path d="M11.13477,1.01758c-0.26304,-0.01259 -0.528,-0.00875 -0.79687,0.01563c-2.22436,0.20069 -4.00167,1.76087 -4.72852,3.78711c-1.71233,0.35652 -3.1721,1.48454 -3.9375,3.13672c-0.93789,2.02702 -0.47459,4.34373 0.91602,5.98633c-0.54763,1.66186 -0.30227,3.49111 0.74414,4.97852c1.28618,1.82589 3.52454,2.58282 5.64258,2.19922c1.16505,1.30652 2.87295,2.00936 4.6875,1.8457c2.22441,-0.2007 4.0017,-1.7608 4.72852,-3.78711c1.71235,-0.35654 3.17321,-1.4837 3.93945,-3.13672c0.93809,-2.0267 0.47529,-4.34583 -0.91602,-5.98828c0.54663,-1.66125 0.29983,-3.48985 -0.74609,-4.97656c-1.28618,-1.82589 -3.52454,-2.58282 -5.64258,-2.19922c-0.99242,-1.11291 -2.37852,-1.78893 -3.89062,-1.86133zM11.02539,2.51367c0.89653,0.03518 1.7296,0.36092 2.40625,0.9082c-0.11306,0.05604 -0.23154,0.09454 -0.3418,0.1582l-4.01367,2.31641c-0.306,0.176 -0.496,0.50247 -0.5,0.85547l-0.05859,5.48633l-1.76758,-1.04883v-4.4043c0,-2.136 1.55759,-4.04291 3.68359,-4.25391c0.19937,-0.01975 0.39689,-0.02523 0.5918,-0.01758zM16.125,4.25586c1.27358,0.00756 2.51484,0.5693 3.29297,1.6543c0.65289,0.90943 0.89227,1.99184 0.72852,3.03711c-0.10507,-0.06991 -0.19832,-0.15312 -0.30859,-0.2168l-4.01172,-2.31641c-0.306,-0.176 -0.68224,-0.17886 -0.99023,-0.00586l-4.7832,2.69531l0.02344,-2.05469l3.81445,-2.20117c0.69375,-0.4005 1.47022,-0.59633 2.23438,-0.5918zM5.2832,6.47266c-0.008,0.12587 -0.0332,0.24774 -0.0332,0.375v4.63281c0,0.353 0.18623,0.67938 0.49023,0.85938l4.72461,2.79688l-1.79102,1.00586l-3.81445,-2.20312c-1.85,-1.068 -2.7228,-3.37236 -1.8418,-5.31836c0.46198,-1.02041 1.27879,-1.76751 2.26562,-2.14844zM15.32617,7.85742l3.81445,2.20313c1.85,1.068 2.72475,3.37236 1.84375,5.31836c-0.46209,1.02065 -1.28043,1.7676 -2.26758,2.14844c0.00797,-0.12565 0.0332,-0.24797 0.0332,-0.375v-4.63086c0,-0.354 -0.18623,-0.68133 -0.49023,-0.86133l-4.72461,-2.79687zM12.02539,9.71094l1.96875,1.16797l-0.02734,2.28906l-1.99219,1.11914l-1.96875,-1.16601l0.02539,-2.28906zM15.48242,11.76172l1.76758,1.04883v4.4043c0,2.136 -1.55759,4.04291 -3.68359,4.25391c-1.11644,0.11059 -2.17429,-0.22435 -2.99805,-0.89062c0.11306,-0.05604 0.23154,-0.09454 0.3418,-0.1582l4.01367,-2.31641c0.306,-0.176 0.496,-0.50247 0.5,-0.85547zM13.94727,14.89648l-0.02344,2.05469l-3.81445,2.20117c-1.85,1.068 -4.28234,0.6735 -5.52734,-1.0625c-0.65289,-0.90943 -0.89227,-1.99184 -0.72852,-3.03711c0.10521,0.07006 0.19816,0.15299 0.30859,0.2168l4.01172,2.31641c0.306,0.176 0.68223,0.17886 0.99023,0.00586z"></path></g></g></svg>

packages/components/nodes/chains/ConversationalRetrievalQAChain/ConversationalRetrievalQAChain.ts

@@ -1,32 +1,21 @@
 import { BaseLanguageModel } from 'langchain/base_language'
 import { ICommonObject, IMessage, INode, INodeData, INodeParams } from '../../../src/Interface'
 import { getBaseClasses } from '../../../src/utils'
-import { ConversationalRetrievalQAChain } from 'langchain/chains'
-import { AIMessage, BaseRetriever, HumanMessage } from 'langchain/schema'
-import { BaseChatMemory, BufferMemory, ChatMessageHistory } from 'langchain/memory'
+import { ConversationalRetrievalQAChain, QAChainParams } from 'langchain/chains'
+import { AIMessage, HumanMessage } from 'langchain/schema'
+import { BaseRetriever } from 'langchain/schema/retriever'
+import { BaseChatMemory, BufferMemory, ChatMessageHistory, BufferMemoryInput } from 'langchain/memory'
 import { PromptTemplate } from 'langchain/prompts'
 import { ConsoleCallbackHandler, CustomChainHandler } from '../../../src/handler'
-
-const default_qa_template = `Use the following pieces of context to answer the question at the end, in its original language. If you don't know the answer, just say that you don't know in its original language, don't try to make up an answer.
-
-{context}
-
-Question: {question}
-Helpful Answer:`
-
-const qa_template = `Use the following pieces of context to answer the question at the end, in its original language.
-
-{context}
-
-Question: {question}
-Helpful Answer:`
-
-const CUSTOM_QUESTION_GENERATOR_CHAIN_PROMPT = `Given the following conversation and a follow up question, rephrase the follow up question to be a standalone question, in its original language. include it in the standalone question.
-
-Chat History:
-{chat_history}
-Follow Up Input: {question}
-Standalone question:`
+import {
+    default_map_reduce_template,
+    default_qa_template,
+    qa_template,
+    map_reduce_template,
+    CUSTOM_QUESTION_GENERATOR_CHAIN_PROMPT,
+    refine_question_template,
+    refine_template
+} from './prompts'
 
 class ConversationalRetrievalQAChain_Chains implements INode {
     label: string
@@ -60,9 +49,9 @@ class ConversationalRetrievalQAChain_Chains implements INode {
             {
                 label: 'Memory',
                 name: 'memory',
-                type: 'DynamoDBChatMemory | RedisBackedChatMemory | ZepMemory',
+                type: 'BaseMemory',
                 optional: true,
-                description: 'If no memory connected, default BufferMemory will be used'
+                description: 'If left empty, a default BufferMemory will be used'
             },
             {
                 label: 'Return Source Documents',
@@ -118,28 +107,54 @@ class ConversationalRetrievalQAChain_Chains implements INode {
 
         const obj: any = {
             verbose: process.env.DEBUG === 'true' ? true : false,
-            qaChainOptions: {
-                type: 'stuff',
-                prompt: PromptTemplate.fromTemplate(systemMessagePrompt ? `${systemMessagePrompt}\n${qa_template}` : default_qa_template)
-            },
             questionGeneratorChainOptions: {
                 template: CUSTOM_QUESTION_GENERATOR_CHAIN_PROMPT
             }
         }
         if (returnSourceDocuments) obj.returnSourceDocuments = returnSourceDocuments
-        if (chainOption) obj.qaChainOptions = { ...obj.qaChainOptions, type: chainOption }
+        if (chainOption === 'map_reduce') {
+            obj.qaChainOptions = {
+                type: 'map_reduce',
+                combinePrompt: PromptTemplate.fromTemplate(
+                    systemMessagePrompt ? `${systemMessagePrompt}\n${map_reduce_template}` : default_map_reduce_template
+                )
+            } as QAChainParams
+        } else if (chainOption === 'refine') {
+            const qprompt = new PromptTemplate({
+                inputVariables: ['context', 'question'],
+                template: refine_question_template(systemMessagePrompt)
+            })
+            const rprompt = new PromptTemplate({
+                inputVariables: ['context', 'question', 'existing_answer'],
+                template: refine_template
+            })
+            obj.qaChainOptions = {
+                type: 'refine',
+                questionPrompt: qprompt,
+                refinePrompt: rprompt
+            } as QAChainParams
+        } else {
+            obj.qaChainOptions = {
+                type: 'stuff',
+                prompt: PromptTemplate.fromTemplate(systemMessagePrompt ? `${systemMessagePrompt}\n${qa_template}` : default_qa_template)
+            } as QAChainParams
+        }
+
         if (memory) {
             memory.inputKey = 'question'
-            memory.outputKey = 'text'
             memory.memoryKey = 'chat_history'
+            if (chainOption === 'refine') memory.outputKey = 'output_text'
+            else memory.outputKey = 'text'
             obj.memory = memory
         } else {
-            obj.memory = new BufferMemory({
+            const fields: BufferMemoryInput = {
                 memoryKey: 'chat_history',
                 inputKey: 'question',
-                outputKey: 'text',
                 returnMessages: true
-            })
+            }
+            if (chainOption === 'refine') fields.outputKey = 'output_text'
+            else fields.outputKey = 'text'
+            obj.memory = new BufferMemory(fields)
         }
 
         const chain = ConversationalRetrievalQAChain.fromLLM(model, vectorStoreRetriever, obj)
@@ -150,6 +165,7 @@ class ConversationalRetrievalQAChain_Chains implements INode {
         const chain = nodeData.instance as ConversationalRetrievalQAChain
         const returnSourceDocuments = nodeData.inputs?.returnSourceDocuments as boolean
         const memory = nodeData.inputs?.memory
+        const chainOption = nodeData.inputs?.chainOption as string
 
         let model = nodeData.inputs?.model
 
@@ -179,8 +195,22 @@ class ConversationalRetrievalQAChain_Chains implements INode {
         const loggerHandler = new ConsoleCallbackHandler(options.logger)
 
         if (options.socketIO && options.socketIOClientId) {
-            const handler = new CustomChainHandler(options.socketIO, options.socketIOClientId, undefined, returnSourceDocuments)
+            const handler = new CustomChainHandler(
+                options.socketIO,
+                options.socketIOClientId,
+                chainOption === 'refine' ? 4 : undefined,
+                returnSourceDocuments
+            )
             const res = await chain.call(obj, [loggerHandler, handler])
+            if (chainOption === 'refine') {
+                if (res.output_text && res.sourceDocuments) {
+                    return {
+                        text: res.output_text,
+                        sourceDocuments: res.sourceDocuments
+                    }
+                }
+                return res?.output_text
+            }
             if (res.text && res.sourceDocuments) return res
             return res?.text
         } else {

packages/components/nodes/chains/ConversationalRetrievalQAChain/prompts.ts

@@ -0,0 +1,64 @@
+export const default_qa_template = `Use the following pieces of context to answer the question at the end. If you don't know the answer, just say that you don't know, don't try to make up an answer.
+
+{context}
+
+Question: {question}
+Helpful Answer:`
+
+export const qa_template = `Use the following pieces of context to answer the question at the end.
+
+{context}
+
+Question: {question}
+Helpful Answer:`
+
+export const default_map_reduce_template = `Given the following extracted parts of a long document and a question, create a final answer. 
+If you don't know the answer, just say that you don't know. Don't try to make up an answer.
+
+{summaries}
+
+Question: {question}
+Helpful Answer:`
+
+export const map_reduce_template = `Given the following extracted parts of a long document and a question, create a final answer. 
+
+{summaries}
+
+Question: {question}
+Helpful Answer:`
+
+export const refine_question_template = (sysPrompt?: string) => {
+    let returnPrompt = ''
+    if (sysPrompt)
+        returnPrompt = `Context information is below. 
+---------------------
+{context}
+---------------------
+Given the context information and not prior knowledge, ${sysPrompt}
+Answer the question: {question}.
+Answer:`
+    if (!sysPrompt)
+        returnPrompt = `Context information is below. 
+---------------------
+{context}
+---------------------
+Given the context information and not prior knowledge, answer the question: {question}.
+Answer:`
+    return returnPrompt
+}
+
+export const refine_template = `The original question is as follows: {question}
+We have provided an existing answer: {existing_answer}
+We have the opportunity to refine the existing answer (only if needed) with some more context below.
+------------
+{context}
+------------
+Given the new context, refine the original answer to better answer the question. 
+If you can't find answer from the context, return the original answer.`
+
+export const CUSTOM_QUESTION_GENERATOR_CHAIN_PROMPT = `Given the following conversation and a follow up question, rephrase the follow up question to be a standalone question, answer in the same language as the follow up question. include it in the standalone question.
+
+Chat History:
+{chat_history}
+Follow Up Input: {question}
+Standalone question:`

packages/components/nodes/chains/LLMChain/LLMChain.ts

@@ -1,5 +1,5 @@
 import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
-import { getBaseClasses } from '../../../src/utils'
+import { getBaseClasses, handleEscapeCharacters } from '../../../src/utils'
 import { LLMChain } from 'langchain/chains'
 import { BaseLanguageModel } from 'langchain/base_language'
 import { ConsoleCallbackHandler, CustomChainHandler } from '../../../src/handler'
@@ -73,7 +73,12 @@ class LLMChain_Chains implements INode {
             console.log('\x1b[92m\x1b[1m\n*****OUTPUT PREDICTION*****\n\x1b[0m\x1b[0m')
             // eslint-disable-next-line no-console
             console.log(res)
-            return res
+            /**
+             * Apply string transformation to convert special chars:
+             * FROM: hello i am ben\n\n\thow are you?
+             * TO: hello i am benFLOWISE_NEWLINEFLOWISE_NEWLINEFLOWISE_TABhow are you?
+             */
+            return handleEscapeCharacters(res, false)
         }
     }
 
@@ -81,7 +86,6 @@ class LLMChain_Chains implements INode {
         const inputVariables = nodeData.instance.prompt.inputVariables as string[] // ["product"]
         const chain = nodeData.instance as LLMChain
         const promptValues = nodeData.inputs?.prompt.promptValues as ICommonObject
-
         const res = await runPrediction(inputVariables, chain, input, promptValues, options)
         // eslint-disable-next-line no-console
         console.log('\x1b[93m\x1b[1m\n*****FINAL RESULT*****\n\x1b[0m\x1b[0m')
@@ -95,7 +99,7 @@ const runPrediction = async (
     inputVariables: string[],
     chain: LLMChain,
     input: string,
-    promptValues: ICommonObject,
+    promptValuesRaw: ICommonObject,
     options: ICommonObject
 ) => {
     const loggerHandler = new ConsoleCallbackHandler(options.logger)
@@ -103,6 +107,13 @@ const runPrediction = async (
     const socketIO = isStreaming ? options.socketIO : undefined
     const socketIOClientId = isStreaming ? options.socketIOClientId : ''
 
+    /**
+     * Apply string transformation to reverse converted special chars:
+     * FROM: { "value": "hello i am benFLOWISE_NEWLINEFLOWISE_NEWLINEFLOWISE_TABhow are you?" }
+     * TO: { "value": "hello i am ben\n\n\thow are you?" }
+     */
+    const promptValues = handleEscapeCharacters(promptValuesRaw, true)
+
     if (inputVariables.length === 1) {
         if (isStreaming) {
             const handler = new CustomChainHandler(socketIO, socketIOClientId)

packages/components/nodes/chains/RetrievalQAChain/RetrievalQAChain.ts

@@ -1,6 +1,6 @@
 import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
 import { RetrievalQAChain } from 'langchain/chains'
-import { BaseRetriever } from 'langchain/schema'
+import { BaseRetriever } from 'langchain/schema/retriever'
 import { getBaseClasses } from '../../../src/utils'
 import { BaseLanguageModel } from 'langchain/base_language'
 import { ConsoleCallbackHandler, CustomChainHandler } from '../../../src/handler'

packages/components/nodes/chatmodels/ChatOpenAI/ChatOpenAI.ts

@@ -17,7 +17,7 @@ class ChatOpenAI_ChatModels implements INode {
         this.label = 'ChatOpenAI'
         this.name = 'chatOpenAI'
         this.type = 'ChatOpenAI'
-        this.icon = 'openai.svg'
+        this.icon = 'openai.png'
         this.category = 'Chat Models'
         this.description = 'Wrapper around OpenAI large language models that use the Chat endpoint'
         this.baseClasses = [this.type, ...getBaseClasses(ChatOpenAI)]

packages/components/nodes/chatmodels/ChatOpenAI/openai.svg

@@ -1 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0,0,256,256" width="96px" height="96px" fill-rule="nonzero"><g fill="#00a67e" fill-rule="nonzero" stroke="none" stroke-width="1" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="10" stroke-dasharray="" stroke-dashoffset="0" font-family="none" font-weight="none" font-size="none" text-anchor="none" style="mix-blend-mode: normal"><g transform="scale(10.66667,10.66667)"><path d="M11.13477,1.01758c-0.26304,-0.01259 -0.528,-0.00875 -0.79687,0.01563c-2.22436,0.20069 -4.00167,1.76087 -4.72852,3.78711c-1.71233,0.35652 -3.1721,1.48454 -3.9375,3.13672c-0.93789,2.02702 -0.47459,4.34373 0.91602,5.98633c-0.54763,1.66186 -0.30227,3.49111 0.74414,4.97852c1.28618,1.82589 3.52454,2.58282 5.64258,2.19922c1.16505,1.30652 2.87295,2.00936 4.6875,1.8457c2.22441,-0.2007 4.0017,-1.7608 4.72852,-3.78711c1.71235,-0.35654 3.17321,-1.4837 3.93945,-3.13672c0.93809,-2.0267 0.47529,-4.34583 -0.91602,-5.98828c0.54663,-1.66125 0.29983,-3.48985 -0.74609,-4.97656c-1.28618,-1.82589 -3.52454,-2.58282 -5.64258,-2.19922c-0.99242,-1.11291 -2.37852,-1.78893 -3.89062,-1.86133zM11.02539,2.51367c0.89653,0.03518 1.7296,0.36092 2.40625,0.9082c-0.11306,0.05604 -0.23154,0.09454 -0.3418,0.1582l-4.01367,2.31641c-0.306,0.176 -0.496,0.50247 -0.5,0.85547l-0.05859,5.48633l-1.76758,-1.04883v-4.4043c0,-2.136 1.55759,-4.04291 3.68359,-4.25391c0.19937,-0.01975 0.39689,-0.02523 0.5918,-0.01758zM16.125,4.25586c1.27358,0.00756 2.51484,0.5693 3.29297,1.6543c0.65289,0.90943 0.89227,1.99184 0.72852,3.03711c-0.10507,-0.06991 -0.19832,-0.15312 -0.30859,-0.2168l-4.01172,-2.31641c-0.306,-0.176 -0.68224,-0.17886 -0.99023,-0.00586l-4.7832,2.69531l0.02344,-2.05469l3.81445,-2.20117c0.69375,-0.4005 1.47022,-0.59633 2.23438,-0.5918zM5.2832,6.47266c-0.008,0.12587 -0.0332,0.24774 -0.0332,0.375v4.63281c0,0.353 0.18623,0.67938 0.49023,0.85938l4.72461,2.79688l-1.79102,1.00586l-3.81445,-2.20312c-1.85,-1.068 -2.7228,-3.37236 -1.8418,-5.31836c0.46198,-1.02041 1.27879,-1.76751 2.26562,-2.14844zM15.32617,7.85742l3.81445,2.20313c1.85,1.068 2.72475,3.37236 1.84375,5.31836c-0.46209,1.02065 -1.28043,1.7676 -2.26758,2.14844c0.00797,-0.12565 0.0332,-0.24797 0.0332,-0.375v-4.63086c0,-0.354 -0.18623,-0.68133 -0.49023,-0.86133l-4.72461,-2.79687zM12.02539,9.71094l1.96875,1.16797l-0.02734,2.28906l-1.99219,1.11914l-1.96875,-1.16601l0.02539,-2.28906zM15.48242,11.76172l1.76758,1.04883v4.4043c0,2.136 -1.55759,4.04291 -3.68359,4.25391c-1.11644,0.11059 -2.17429,-0.22435 -2.99805,-0.89062c0.11306,-0.05604 0.23154,-0.09454 0.3418,-0.1582l4.01367,-2.31641c0.306,-0.176 0.496,-0.50247 0.5,-0.85547zM13.94727,14.89648l-0.02344,2.05469l-3.81445,2.20117c-1.85,1.068 -4.28234,0.6735 -5.52734,-1.0625c-0.65289,-0.90943 -0.89227,-1.99184 -0.72852,-3.03711c0.10521,0.07006 0.19816,0.15299 0.30859,0.2168l4.01172,2.31641c0.306,0.176 0.68223,0.17886 0.99023,0.00586z"></path></g></g></svg>

packages/components/nodes/documentloaders/API/APILoader.ts

@@ -0,0 +1,198 @@
+import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
+import { TextSplitter } from 'langchain/text_splitter'
+import { BaseDocumentLoader } from 'langchain/document_loaders/base'
+import { Document } from 'langchain/document'
+import axios, { AxiosRequestConfig } from 'axios'
+
+class API_DocumentLoaders implements INode {
+    label: string
+    name: string
+    description: string
+    type: string
+    icon: string
+    category: string
+    baseClasses: string[]
+    inputs?: INodeParams[]
+
+    constructor() {
+        this.label = 'API Loader'
+        this.name = 'apiLoader'
+        this.type = 'Document'
+        this.icon = 'api-loader.png'
+        this.category = 'Document Loaders'
+        this.description = `Load data from an API`
+        this.baseClasses = [this.type]
+        this.inputs = [
+            {
+                label: 'Text Splitter',
+                name: 'textSplitter',
+                type: 'TextSplitter',
+                optional: true
+            },
+            {
+                label: 'Method',
+                name: 'method',
+                type: 'options',
+                options: [
+                    {
+                        label: 'GET',
+                        name: 'GET'
+                    },
+                    {
+                        label: 'POST',
+                        name: 'POST'
+                    }
+                ]
+            },
+            {
+                label: 'URL',
+                name: 'url',
+                type: 'string'
+            },
+            {
+                label: 'Headers',
+                name: 'headers',
+                type: 'json',
+                additionalParams: true,
+                optional: true
+            },
+            {
+                label: 'Body',
+                name: 'body',
+                type: 'json',
+                description:
+                    'JSON body for the POST request. If not specified, agent will try to figure out itself from AIPlugin if provided',
+                additionalParams: true,
+                optional: true
+            }
+        ]
+    }
+    async init(nodeData: INodeData): Promise<any> {
+        const headers = nodeData.inputs?.headers as string
+        const url = nodeData.inputs?.url as string
+        const body = nodeData.inputs?.body as string
+        const method = nodeData.inputs?.method as string
+        const textSplitter = nodeData.inputs?.textSplitter as TextSplitter
+        const metadata = nodeData.inputs?.metadata
+
+        const options: ApiLoaderParams = {
+            url,
+            method
+        }
+
+        if (headers) {
+            const parsedHeaders = typeof headers === 'object' ? headers : JSON.parse(headers)
+            options.headers = parsedHeaders
+        }
+
+        if (body) {
+            const parsedBody = typeof body === 'object' ? body : JSON.parse(body)
+            options.body = parsedBody
+        }
+
+        const loader = new ApiLoader(options)
+
+        let docs = []
+
+        if (textSplitter) {
+            docs = await loader.loadAndSplit(textSplitter)
+        } else {
+            docs = await loader.load()
+        }
+
+        if (metadata) {
+            const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata)
+            let finaldocs = []
+            for (const doc of docs) {
+                const newdoc = {
+                    ...doc,
+                    metadata: {
+                        ...doc.metadata,
+                        ...parsedMetadata
+                    }
+                }
+                finaldocs.push(newdoc)
+            }
+            return finaldocs
+        }
+
+        return docs
+    }
+}
+
+interface ApiLoaderParams {
+    url: string
+    method: string
+    headers?: ICommonObject
+    body?: ICommonObject
+}
+
+class ApiLoader extends BaseDocumentLoader {
+    public readonly url: string
+
+    public readonly headers?: ICommonObject
+
+    public readonly body?: ICommonObject
+
+    public readonly method: string
+
+    constructor({ url, headers, body, method }: ApiLoaderParams) {
+        super()
+        this.url = url
+        this.headers = headers
+        this.body = body
+        this.method = method
+    }
+
+    public async load(): Promise<Document[]> {
+        if (this.method === 'POST') {
+            return this.executePostRequest(this.url, this.headers, this.body)
+        } else {
+            return this.executeGetRequest(this.url, this.headers)
+        }
+    }
+
+    protected async executeGetRequest(url: string, headers?: ICommonObject): Promise<Document[]> {
+        try {
+            const config: AxiosRequestConfig = {}
+            if (headers) {
+                config.headers = headers
+            }
+            const response = await axios.get(url, config)
+            const responseJsonString = JSON.stringify(response.data, null, 2)
+            const doc = new Document({
+                pageContent: responseJsonString,
+                metadata: {
+                    url
+                }
+            })
+            return [doc]
+        } catch (error) {
+            throw new Error(`Failed to fetch ${url}: ${error}`)
+        }
+    }
+
+    protected async executePostRequest(url: string, headers?: ICommonObject, body?: ICommonObject): Promise<Document[]> {
+        try {
+            const config: AxiosRequestConfig = {}
+            if (headers) {
+                config.headers = headers
+            }
+            const response = await axios.post(url, body ?? {}, config)
+            const responseJsonString = JSON.stringify(response.data, null, 2)
+            const doc = new Document({
+                pageContent: responseJsonString,
+                metadata: {
+                    url
+                }
+            })
+            return [doc]
+        } catch (error) {
+            throw new Error(`Failed to post ${url}: ${error}`)
+        }
+    }
+}
+
+module.exports = {
+    nodeClass: API_DocumentLoaders
+}

packages/components/nodes/documentloaders/Airtable/Airtable.ts

@@ -0,0 +1,229 @@
+import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
+import { TextSplitter } from 'langchain/text_splitter'
+import { BaseDocumentLoader } from 'langchain/document_loaders/base'
+import { Document } from 'langchain/document'
+import axios from 'axios'
+import { getCredentialData, getCredentialParam } from '../../../src/utils'
+
+class Airtable_DocumentLoaders implements INode {
+    label: string
+    name: string
+    description: string
+    type: string
+    icon: string
+    category: string
+    baseClasses: string[]
+    credential: INodeParams
+    inputs?: INodeParams[]
+
+    constructor() {
+        this.label = 'Airtable'
+        this.name = 'airtable'
+        this.type = 'Document'
+        this.icon = 'airtable.svg'
+        this.category = 'Document Loaders'
+        this.description = `Load data from Airtable table`
+        this.baseClasses = [this.type]
+        this.credential = {
+            label: 'Connect Credential',
+            name: 'credential',
+            type: 'credential',
+            credentialNames: ['airtableApi']
+        }
+        this.inputs = [
+            {
+                label: 'Text Splitter',
+                name: 'textSplitter',
+                type: 'TextSplitter',
+                optional: true
+            },
+            {
+                label: 'Base Id',
+                name: 'baseId',
+                type: 'string',
+                placeholder: 'app11RobdGoX0YNsC',
+                description:
+                    'If your table URL looks like: https://airtable.com/app11RobdGoX0YNsC/tblJdmvbrgizbYICO/viw9UrP77Id0CE4ee, app11RovdGoX0YNsC is the base id'
+            },
+            {
+                label: 'Table Id',
+                name: 'tableId',
+                type: 'string',
+                placeholder: 'tblJdmvbrgizbYICO',
+                description:
+                    'If your table URL looks like: https://airtable.com/app11RobdGoX0YNsC/tblJdmvbrgizbYICO/viw9UrP77Id0CE4ee, tblJdmvbrgizbYICO is the table id'
+            },
+            {
+                label: 'Return All',
+                name: 'returnAll',
+                type: 'boolean',
+                default: true,
+                additionalParams: true,
+                description: 'If all results should be returned or only up to a given limit'
+            },
+            {
+                label: 'Limit',
+                name: 'limit',
+                type: 'number',
+                default: 100,
+                step: 1,
+                additionalParams: true,
+                description: 'Number of results to return'
+            },
+            {
+                label: 'Metadata',
+                name: 'metadata',
+                type: 'json',
+                optional: true,
+                additionalParams: true
+            }
+        ]
+    }
+    async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
+        const baseId = nodeData.inputs?.baseId as string
+        const tableId = nodeData.inputs?.tableId as string
+        const returnAll = nodeData.inputs?.returnAll as boolean
+        const limit = nodeData.inputs?.limit as string
+        const textSplitter = nodeData.inputs?.textSplitter as TextSplitter
+        const metadata = nodeData.inputs?.metadata
+
+        const credentialData = await getCredentialData(nodeData.credential ?? '', options)
+        const accessToken = getCredentialParam('accessToken', credentialData, nodeData)
+
+        const airtableOptions: AirtableLoaderParams = {
+            baseId,
+            tableId,
+            returnAll,
+            accessToken,
+            limit: limit ? parseInt(limit, 10) : 100
+        }
+
+        const loader = new AirtableLoader(airtableOptions)
+
+        let docs = []
+
+        if (textSplitter) {
+            docs = await loader.loadAndSplit(textSplitter)
+        } else {
+            docs = await loader.load()
+        }
+
+        if (metadata) {
+            const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata)
+            let finaldocs = []
+            for (const doc of docs) {
+                const newdoc = {
+                    ...doc,
+                    metadata: {
+                        ...doc.metadata,
+                        ...parsedMetadata
+                    }
+                }
+                finaldocs.push(newdoc)
+            }
+            return finaldocs
+        }
+
+        return docs
+    }
+}
+
+interface AirtableLoaderParams {
+    baseId: string
+    tableId: string
+    accessToken: string
+    limit?: number
+    returnAll?: boolean
+}
+
+interface AirtableLoaderResponse {
+    records: AirtableLoaderPage[]
+    offset?: string
+}
+
+interface AirtableLoaderPage {
+    id: string
+    createdTime: string
+    fields: ICommonObject
+}
+
+class AirtableLoader extends BaseDocumentLoader {
+    public readonly baseId: string
+
+    public readonly tableId: string
+
+    public readonly accessToken: string
+
+    public readonly limit: number
+
+    public readonly returnAll: boolean
+
+    constructor({ baseId, tableId, accessToken, limit = 100, returnAll = false }: AirtableLoaderParams) {
+        super()
+        this.baseId = baseId
+        this.tableId = tableId
+        this.accessToken = accessToken
+        this.limit = limit
+        this.returnAll = returnAll
+    }
+
+    public async load(): Promise<Document[]> {
+        if (this.returnAll) {
+            return this.loadAll()
+        }
+        return this.loadLimit()
+    }
+
+    protected async fetchAirtableData(url: string, params: ICommonObject): Promise<AirtableLoaderResponse> {
+        try {
+            const headers = {
+                Authorization: `Bearer ${this.accessToken}`,
+                'Content-Type': 'application/json',
+                Accept: 'application/json'
+            }
+            const response = await axios.get(url, { params, headers })
+            return response.data
+        } catch (error) {
+            throw new Error(`Failed to fetch ${url} from Airtable: ${error}`)
+        }
+    }
+
+    private createDocumentFromPage(page: AirtableLoaderPage): Document {
+        // Generate the URL
+        const pageUrl = `https://api.airtable.com/v0/${this.baseId}/${this.tableId}/${page.id}`
+
+        // Return a langchain document
+        return new Document({
+            pageContent: JSON.stringify(page.fields, null, 2),
+            metadata: {
+                url: pageUrl
+            }
+        })
+    }
+
+    private async loadLimit(): Promise<Document[]> {
+        const params = { maxRecords: this.limit }
+        const data = await this.fetchAirtableData(`https://api.airtable.com/v0/${this.baseId}/${this.tableId}`, params)
+        if (data.records.length === 0) {
+            return []
+        }
+        return data.records.map((page) => this.createDocumentFromPage(page))
+    }
+
+    private async loadAll(): Promise<Document[]> {
+        const params: ICommonObject = { pageSize: 100 }
+        let data: AirtableLoaderResponse
+        let returnPages: AirtableLoaderPage[] = []
+
+        do {
+            data = await this.fetchAirtableData(`https://api.airtable.com/v0/${this.baseId}/${this.tableId}`, params)
+            returnPages.push.apply(returnPages, data.records)
+            params.offset = data.offset
+        } while (data.offset !== undefined)
+        return returnPages.map((page) => this.createDocumentFromPage(page))
+    }
+}
+
+module.exports = {
+    nodeClass: Airtable_DocumentLoaders
+}

packages/components/nodes/documentloaders/Airtable/airtable.svg

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<svg width="256px" height="215px" viewBox="0 0 256 215" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" preserveAspectRatio="xMidYMid">
+	<g>
+		<path d="M114.25873,2.70101695 L18.8604023,42.1756384 C13.5552723,44.3711638 13.6102328,51.9065311 18.9486282,54.0225085 L114.746142,92.0117514 C123.163769,95.3498757 132.537419,95.3498757 140.9536,92.0117514 L236.75256,54.0225085 C242.08951,51.9065311 242.145916,44.3711638 236.83934,42.1756384 L141.442459,2.70101695 C132.738459,-0.900338983 122.961284,-0.900338983 114.25873,2.70101695" fill="#FFBF00"></path>
+		<path d="M136.349071,112.756863 L136.349071,207.659101 C136.349071,212.173089 140.900664,215.263892 145.096461,213.600615 L251.844122,172.166219 C254.281184,171.200072 255.879376,168.845451 255.879376,166.224705 L255.879376,71.3224678 C255.879376,66.8084791 251.327783,63.7176768 247.131986,65.3809537 L140.384325,106.815349 C137.94871,107.781496 136.349071,110.136118 136.349071,112.756863" fill="#26B5F8"></path>
+		<path d="M111.422771,117.65355 L79.742409,132.949912 L76.5257763,134.504714 L9.65047684,166.548104 C5.4112904,168.593211 0.000578531073,165.503855 0.000578531073,160.794612 L0.000578531073,71.7210757 C0.000578531073,70.0173017 0.874160452,68.5463864 2.04568588,67.4384994 C2.53454463,66.9481944 3.08848814,66.5446689 3.66412655,66.2250305 C5.26231864,65.2661153 7.54173107,65.0101153 9.47981017,65.7766689 L110.890522,105.957098 C116.045234,108.002206 116.450206,115.225166 111.422771,117.65355" fill="#ED3049"></path>
+		<path d="M111.422771,117.65355 L79.742409,132.949912 L2.04568588,67.4384994 C2.53454463,66.9481944 3.08848814,66.5446689 3.66412655,66.2250305 C5.26231864,65.2661153 7.54173107,65.0101153 9.47981017,65.7766689 L110.890522,105.957098 C116.045234,108.002206 116.450206,115.225166 111.422771,117.65355" fill-opacity="0.25" fill="#000000"></path>
+	</g>
+</svg>

packages/components/nodes/documentloaders/Notion/NotionDB.ts

@@ -39,8 +39,7 @@ class NotionDB_DocumentLoaders implements INode {
                 label: 'Notion Database Id',
                 name: 'databaseId',
                 type: 'string',
-                description:
-                    'If your URL looks like - https://www.notion.so/<long_hash_1>?v=<long_hash_2>, then <long_hash_1> is the database ID'
+                description: 'If your URL looks like - https://www.notion.so/abcdefh?v=long_hash_2, then abcdefh is the database ID'
             },
             {
                 label: 'Metadata',

packages/components/nodes/embeddings/HuggingFaceInferenceEmbedding/HuggingFaceInferenceEmbedding.ts

@@ -32,6 +32,8 @@ class HuggingFaceInferenceEmbedding_Embeddings implements INode {
                 label: 'Model',
                 name: 'modelName',
                 type: 'string',
+                description: 'If using own inference endpoint, leave this blank',
+                placeholder: 'sentence-transformers/distilbert-base-nli-mean-tokens',
                 optional: true
             },
             {

packages/components/nodes/embeddings/HuggingFaceInferenceEmbedding/core.ts

@@ -30,18 +30,26 @@ export class HuggingFaceInferenceEmbeddings extends Embeddings implements Huggin
     async _embed(texts: string[]): Promise<number[][]> {
         // replace newlines, which can negatively affect performance.
         const clean = texts.map((text) => text.replace(/\n/g, ' '))
+        const hf = new HfInference(this.apiKey)
         const obj: any = {
             inputs: clean
         }
-        if (!this.endpoint) obj.model = this.model
-        return this.caller.call(() => this.client.featureExtraction(obj)) as Promise<number[][]>
+        if (this.endpoint) {
+            hf.endpoint(this.endpoint)
+        } else {
+            obj.model = this.model
+        }
+
+        const res = await this.caller.callWithOptions({}, hf.featureExtraction.bind(hf), obj)
+        return res as number[][]
     }
 
-    embedQuery(document: string): Promise<number[]> {
-        return this._embed([document]).then((embeddings) => embeddings[0])
+    async embedQuery(document: string): Promise<number[]> {
+        const res = await this._embed([document])
+        return res[0]
     }
 
-    embedDocuments(documents: string[]): Promise<number[][]> {
+    async embedDocuments(documents: string[]): Promise<number[][]> {
         return this._embed(documents)
     }
 }

packages/components/nodes/embeddings/OpenAIEmbedding/OpenAIEmbedding.ts

@@ -17,7 +17,7 @@ class OpenAIEmbedding_Embeddings implements INode {
         this.label = 'OpenAI Embeddings'
         this.name = 'openAIEmbeddings'
         this.type = 'OpenAIEmbeddings'
-        this.icon = 'openai.svg'
+        this.icon = 'openai.png'
         this.category = 'Embeddings'
         this.description = 'OpenAI API to generate embeddings for a given text'
         this.baseClasses = [this.type, ...getBaseClasses(OpenAIEmbeddings)]

packages/components/nodes/embeddings/OpenAIEmbedding/openai.svg

@@ -1 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0,0,256,256" width="96px" height="96px" fill-rule="nonzero"><g fill="#00a67e" fill-rule="nonzero" stroke="none" stroke-width="1" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="10" stroke-dasharray="" stroke-dashoffset="0" font-family="none" font-weight="none" font-size="none" text-anchor="none" style="mix-blend-mode: normal"><g transform="scale(10.66667,10.66667)"><path d="M11.13477,1.01758c-0.26304,-0.01259 -0.528,-0.00875 -0.79687,0.01563c-2.22436,0.20069 -4.00167,1.76087 -4.72852,3.78711c-1.71233,0.35652 -3.1721,1.48454 -3.9375,3.13672c-0.93789,2.02702 -0.47459,4.34373 0.91602,5.98633c-0.54763,1.66186 -0.30227,3.49111 0.74414,4.97852c1.28618,1.82589 3.52454,2.58282 5.64258,2.19922c1.16505,1.30652 2.87295,2.00936 4.6875,1.8457c2.22441,-0.2007 4.0017,-1.7608 4.72852,-3.78711c1.71235,-0.35654 3.17321,-1.4837 3.93945,-3.13672c0.93809,-2.0267 0.47529,-4.34583 -0.91602,-5.98828c0.54663,-1.66125 0.29983,-3.48985 -0.74609,-4.97656c-1.28618,-1.82589 -3.52454,-2.58282 -5.64258,-2.19922c-0.99242,-1.11291 -2.37852,-1.78893 -3.89062,-1.86133zM11.02539,2.51367c0.89653,0.03518 1.7296,0.36092 2.40625,0.9082c-0.11306,0.05604 -0.23154,0.09454 -0.3418,0.1582l-4.01367,2.31641c-0.306,0.176 -0.496,0.50247 -0.5,0.85547l-0.05859,5.48633l-1.76758,-1.04883v-4.4043c0,-2.136 1.55759,-4.04291 3.68359,-4.25391c0.19937,-0.01975 0.39689,-0.02523 0.5918,-0.01758zM16.125,4.25586c1.27358,0.00756 2.51484,0.5693 3.29297,1.6543c0.65289,0.90943 0.89227,1.99184 0.72852,3.03711c-0.10507,-0.06991 -0.19832,-0.15312 -0.30859,-0.2168l-4.01172,-2.31641c-0.306,-0.176 -0.68224,-0.17886 -0.99023,-0.00586l-4.7832,2.69531l0.02344,-2.05469l3.81445,-2.20117c0.69375,-0.4005 1.47022,-0.59633 2.23438,-0.5918zM5.2832,6.47266c-0.008,0.12587 -0.0332,0.24774 -0.0332,0.375v4.63281c0,0.353 0.18623,0.67938 0.49023,0.85938l4.72461,2.79688l-1.79102,1.00586l-3.81445,-2.20312c-1.85,-1.068 -2.7228,-3.37236 -1.8418,-5.31836c0.46198,-1.02041 1.27879,-1.76751 2.26562,-2.14844zM15.32617,7.85742l3.81445,2.20313c1.85,1.068 2.72475,3.37236 1.84375,5.31836c-0.46209,1.02065 -1.28043,1.7676 -2.26758,2.14844c0.00797,-0.12565 0.0332,-0.24797 0.0332,-0.375v-4.63086c0,-0.354 -0.18623,-0.68133 -0.49023,-0.86133l-4.72461,-2.79687zM12.02539,9.71094l1.96875,1.16797l-0.02734,2.28906l-1.99219,1.11914l-1.96875,-1.16601l0.02539,-2.28906zM15.48242,11.76172l1.76758,1.04883v4.4043c0,2.136 -1.55759,4.04291 -3.68359,4.25391c-1.11644,0.11059 -2.17429,-0.22435 -2.99805,-0.89062c0.11306,-0.05604 0.23154,-0.09454 0.3418,-0.1582l4.01367,-2.31641c0.306,-0.176 0.496,-0.50247 0.5,-0.85547zM13.94727,14.89648l-0.02344,2.05469l-3.81445,2.20117c-1.85,1.068 -4.28234,0.6735 -5.52734,-1.0625c-0.65289,-0.90943 -0.89227,-1.99184 -0.72852,-3.03711c0.10521,0.07006 0.19816,0.15299 0.30859,0.2168l4.01172,2.31641c0.306,0.176 0.68223,0.17886 0.99023,0.00586z"></path></g></g></svg>

packages/components/nodes/llms/OpenAI/OpenAI.ts

@@ -17,7 +17,7 @@ class OpenAI_LLMs implements INode {
         this.label = 'OpenAI'
         this.name = 'openAI'
         this.type = 'OpenAI'
-        this.icon = 'openai.svg'
+        this.icon = 'openai.png'
         this.category = 'LLMs'
         this.description = 'Wrapper around OpenAI large language models'
         this.baseClasses = [this.type, ...getBaseClasses(OpenAI)]

packages/components/nodes/llms/OpenAI/openai.svg

@@ -1 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0,0,256,256" width="96px" height="96px" fill-rule="nonzero"><g fill="#00a67e" fill-rule="nonzero" stroke="none" stroke-width="1" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="10" stroke-dasharray="" stroke-dashoffset="0" font-family="none" font-weight="none" font-size="none" text-anchor="none" style="mix-blend-mode: normal"><g transform="scale(10.66667,10.66667)"><path d="M11.13477,1.01758c-0.26304,-0.01259 -0.528,-0.00875 -0.79687,0.01563c-2.22436,0.20069 -4.00167,1.76087 -4.72852,3.78711c-1.71233,0.35652 -3.1721,1.48454 -3.9375,3.13672c-0.93789,2.02702 -0.47459,4.34373 0.91602,5.98633c-0.54763,1.66186 -0.30227,3.49111 0.74414,4.97852c1.28618,1.82589 3.52454,2.58282 5.64258,2.19922c1.16505,1.30652 2.87295,2.00936 4.6875,1.8457c2.22441,-0.2007 4.0017,-1.7608 4.72852,-3.78711c1.71235,-0.35654 3.17321,-1.4837 3.93945,-3.13672c0.93809,-2.0267 0.47529,-4.34583 -0.91602,-5.98828c0.54663,-1.66125 0.29983,-3.48985 -0.74609,-4.97656c-1.28618,-1.82589 -3.52454,-2.58282 -5.64258,-2.19922c-0.99242,-1.11291 -2.37852,-1.78893 -3.89062,-1.86133zM11.02539,2.51367c0.89653,0.03518 1.7296,0.36092 2.40625,0.9082c-0.11306,0.05604 -0.23154,0.09454 -0.3418,0.1582l-4.01367,2.31641c-0.306,0.176 -0.496,0.50247 -0.5,0.85547l-0.05859,5.48633l-1.76758,-1.04883v-4.4043c0,-2.136 1.55759,-4.04291 3.68359,-4.25391c0.19937,-0.01975 0.39689,-0.02523 0.5918,-0.01758zM16.125,4.25586c1.27358,0.00756 2.51484,0.5693 3.29297,1.6543c0.65289,0.90943 0.89227,1.99184 0.72852,3.03711c-0.10507,-0.06991 -0.19832,-0.15312 -0.30859,-0.2168l-4.01172,-2.31641c-0.306,-0.176 -0.68224,-0.17886 -0.99023,-0.00586l-4.7832,2.69531l0.02344,-2.05469l3.81445,-2.20117c0.69375,-0.4005 1.47022,-0.59633 2.23438,-0.5918zM5.2832,6.47266c-0.008,0.12587 -0.0332,0.24774 -0.0332,0.375v4.63281c0,0.353 0.18623,0.67938 0.49023,0.85938l4.72461,2.79688l-1.79102,1.00586l-3.81445,-2.20312c-1.85,-1.068 -2.7228,-3.37236 -1.8418,-5.31836c0.46198,-1.02041 1.27879,-1.76751 2.26562,-2.14844zM15.32617,7.85742l3.81445,2.20313c1.85,1.068 2.72475,3.37236 1.84375,5.31836c-0.46209,1.02065 -1.28043,1.7676 -2.26758,2.14844c0.00797,-0.12565 0.0332,-0.24797 0.0332,-0.375v-4.63086c0,-0.354 -0.18623,-0.68133 -0.49023,-0.86133l-4.72461,-2.79687zM12.02539,9.71094l1.96875,1.16797l-0.02734,2.28906l-1.99219,1.11914l-1.96875,-1.16601l0.02539,-2.28906zM15.48242,11.76172l1.76758,1.04883v4.4043c0,2.136 -1.55759,4.04291 -3.68359,4.25391c-1.11644,0.11059 -2.17429,-0.22435 -2.99805,-0.89062c0.11306,-0.05604 0.23154,-0.09454 0.3418,-0.1582l4.01367,-2.31641c0.306,-0.176 0.496,-0.50247 0.5,-0.85547zM13.94727,14.89648l-0.02344,2.05469l-3.81445,2.20117c-1.85,1.068 -4.28234,0.6735 -5.52734,-1.0625c-0.65289,-0.90943 -0.89227,-1.99184 -0.72852,-3.03711c0.10521,0.07006 0.19816,0.15299 0.30859,0.2168l4.01172,2.31641c0.306,0.176 0.68223,0.17886 0.99023,0.00586z"></path></g></g></svg>

packages/components/nodes/llms/Replicate/Replicate.ts

@@ -0,0 +1,122 @@
+import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
+import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
+import { Replicate, ReplicateInput } from 'langchain/llms/replicate'
+
+class Replicate_LLMs implements INode {
+    label: string
+    name: string
+    type: string
+    icon: string
+    category: string
+    description: string
+    baseClasses: string[]
+    credential: INodeParams
+    inputs: INodeParams[]
+
+    constructor() {
+        this.label = 'Replicate'
+        this.name = 'replicate'
+        this.type = 'Replicate'
+        this.icon = 'replicate.svg'
+        this.category = 'LLMs'
+        this.description = 'Use Replicate to run open source models on cloud'
+        this.baseClasses = [this.type, 'BaseChatModel', ...getBaseClasses(Replicate)]
+        this.credential = {
+            label: 'Connect Credential',
+            name: 'credential',
+            type: 'credential',
+            credentialNames: ['replicateApi']
+        }
+        this.inputs = [
+            {
+                label: 'Model',
+                name: 'model',
+                type: 'string',
+                placeholder: 'a16z-infra/llama13b-v2-chat:df7690f1994d94e96ad9d568eac121aecf50684a0b0963b25a41cc40061269e5',
+                optional: true
+            },
+            {
+                label: 'Temperature',
+                name: 'temperature',
+                type: 'number',
+                description:
+                    'Adjusts randomness of outputs, greater than 1 is random and 0 is deterministic, 0.75 is a good starting value.',
+                default: 0.7,
+                optional: true
+            },
+            {
+                label: 'Max Tokens',
+                name: 'maxTokens',
+                type: 'number',
+                description: 'Maximum number of tokens to generate. A word is generally 2-3 tokens',
+                optional: true,
+                additionalParams: true
+            },
+            {
+                label: 'Top Probability',
+                name: 'topP',
+                type: 'number',
+                description:
+                    'When decoding text, samples from the top p percentage of most likely tokens; lower to ignore less likely tokens',
+                optional: true,
+                additionalParams: true
+            },
+            {
+                label: 'Repetition Penalty',
+                name: 'repetitionPenalty',
+                type: 'number',
+                description:
+                    'Penalty for repeated words in generated text; 1 is no penalty, values greater than 1 discourage repetition, less than 1 encourage it. (minimum: 0.01; maximum: 5)',
+                optional: true,
+                additionalParams: true
+            },
+            {
+                label: 'Additional Inputs',
+                name: 'additionalInputs',
+                type: 'json',
+                description:
+                    'Each model has different parameters, refer to the specific model accepted inputs. For example: <a target="_blank" href="https://replicate.com/a16z-infra/llama13b-v2-chat/api#inputs">llama13b-v2</a>',
+                additionalParams: true,
+                optional: true
+            }
+        ]
+    }
+
+    async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
+        const modelName = nodeData.inputs?.model as string
+        const temperature = nodeData.inputs?.temperature as string
+        const maxTokens = nodeData.inputs?.maxTokens as string
+        const topP = nodeData.inputs?.topP as string
+        const repetitionPenalty = nodeData.inputs?.repetitionPenalty as string
+        const additionalInputs = nodeData.inputs?.additionalInputs as string
+
+        const credentialData = await getCredentialData(nodeData.credential ?? '', options)
+        const apiKey = getCredentialParam('apiKey', credentialData, nodeData)
+
+        const version = modelName.split(':').pop()
+        const name = modelName.split(':')[0].split('/').pop()
+        const org = modelName.split(':')[0].split('/')[0]
+
+        const obj: ReplicateInput = {
+            model: `${org}/${name}:${version}`,
+            apiKey
+        }
+
+        let inputs: any = {}
+        if (maxTokens) inputs.max_length = parseInt(maxTokens, 10)
+        if (temperature) inputs.temperature = parseFloat(temperature)
+        if (topP) inputs.top_p = parseFloat(topP)
+        if (repetitionPenalty) inputs.repetition_penalty = parseFloat(repetitionPenalty)
+        if (additionalInputs) {
+            const parsedInputs =
+                typeof additionalInputs === 'object' ? additionalInputs : additionalInputs ? JSON.parse(additionalInputs) : {}
+            inputs = { ...inputs, ...parsedInputs }
+        }
+        if (Object.keys(inputs).length) obj.input = inputs
+
+        const model = new Replicate(obj)
+        return model
+    }
+}
+
+module.exports = { nodeClass: Replicate_LLMs }

packages/components/nodes/llms/Replicate/replicate.svg

@@ -0,0 +1,7 @@
+<svg version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px" viewBox="0 0 1000 1000" class="logo" xml:space="preserve">
+  <g>
+    <polygon points="1000,427.6 1000,540.6 603.4,540.6 603.4,1000 477,1000 477,427.6 	"></polygon>
+    <polygon points="1000,213.8 1000,327 364.8,327 364.8,1000 238.4,1000 238.4,213.8 	"></polygon>
+    <polygon points="1000,0 1000,113.2 126.4,113.2 126.4,1000 0,1000 0,0 	"></polygon>
+  </g>
+</svg>

packages/components/nodes/memory/DynamoDb/DynamoDb.ts

@@ -63,39 +63,49 @@ class DynamoDb_Memory implements INode {
             }
         ]
     }
+
     async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
-        const tableName = nodeData.inputs?.tableName as string
-        const partitionKey = nodeData.inputs?.partitionKey as string
-        const sessionId = nodeData.inputs?.sessionId as string
-        const region = nodeData.inputs?.region as string
-        const memoryKey = nodeData.inputs?.memoryKey as string
+        return initalizeDynamoDB(nodeData, options)
+    }
 
-        const chatId = options.chatId
-
-        const credentialData = await getCredentialData(nodeData.credential ?? '', options)
-        const accessKey = getCredentialParam('accessKey', credentialData, nodeData)
-        const secretAccessKey = getCredentialParam('secretAccessKey', credentialData, nodeData)
-
-        const dynamoDb = new DynamoDBChatMessageHistory({
-            tableName,
-            partitionKey,
-            sessionId: sessionId ? sessionId : chatId,
-            config: {
-                region,
-                credentials: {
-                    accessKeyId: accessKey,
-                    secretAccessKey
-                }
-            }
-        })
-
-        const memory = new BufferMemory({
-            memoryKey,
-            chatHistory: dynamoDb,
-            returnMessages: true
-        })
-        return memory
+    async clearSessionMemory(nodeData: INodeData, options: ICommonObject): Promise<void> {
+        const dynamodbMemory = await initalizeDynamoDB(nodeData, options)
+        await dynamodbMemory.clear()
     }
 }
 
+const initalizeDynamoDB = async (nodeData: INodeData, options: ICommonObject): Promise<BufferMemory> => {
+    const tableName = nodeData.inputs?.tableName as string
+    const partitionKey = nodeData.inputs?.partitionKey as string
+    const sessionId = nodeData.inputs?.sessionId as string
+    const region = nodeData.inputs?.region as string
+    const memoryKey = nodeData.inputs?.memoryKey as string
+
+    const chatId = options.chatId
+
+    const credentialData = await getCredentialData(nodeData.credential ?? '', options)
+    const accessKeyId = getCredentialParam('accessKey', credentialData, nodeData)
+    const secretAccessKey = getCredentialParam('secretAccessKey', credentialData, nodeData)
+
+    const dynamoDb = new DynamoDBChatMessageHistory({
+        tableName,
+        partitionKey,
+        sessionId: sessionId ? sessionId : chatId,
+        config: {
+            region,
+            credentials: {
+                accessKeyId,
+                secretAccessKey
+            }
+        }
+    })
+
+    const memory = new BufferMemory({
+        memoryKey,
+        chatHistory: dynamoDb,
+        returnMessages: true
+    })
+    return memory
+}
+
 module.exports = { nodeClass: DynamoDb_Memory }

packages/components/nodes/memory/MotorheadMemory/MotorheadMemory.ts

@@ -58,37 +58,46 @@ class MotorMemory_Memory implements INode {
     }
 
     async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
-        const memoryKey = nodeData.inputs?.memoryKey as string
-        const baseURL = nodeData.inputs?.baseURL as string
-        const sessionId = nodeData.inputs?.sessionId as string
+        return initalizeMotorhead(nodeData, options)
+    }
 
-        const chatId = options?.chatId as string
-
-        const credentialData = await getCredentialData(nodeData.credential ?? '', options)
-        const apiKey = getCredentialParam('apiKey', credentialData, nodeData)
-        const clientId = getCredentialParam('clientId', credentialData, nodeData)
-
-        let obj: MotorheadMemoryInput = {
-            returnMessages: true,
-            sessionId: sessionId ? sessionId : chatId,
-            memoryKey
-        }
-
-        if (baseURL) {
-            obj = {
-                ...obj,
-                url: baseURL
-            }
-        } else {
-            obj = {
-                ...obj,
-                apiKey,
-                clientId
-            }
-        }
-
-        return new MotorheadMemory(obj)
+    async clearSessionMemory(nodeData: INodeData, options: ICommonObject): Promise<void> {
+        const motorhead = await initalizeMotorhead(nodeData, options)
+        await motorhead.clear()
     }
 }
 
+const initalizeMotorhead = async (nodeData: INodeData, options: ICommonObject): Promise<MotorheadMemory> => {
+    const memoryKey = nodeData.inputs?.memoryKey as string
+    const baseURL = nodeData.inputs?.baseURL as string
+    const sessionId = nodeData.inputs?.sessionId as string
+
+    const chatId = options?.chatId as string
+
+    const credentialData = await getCredentialData(nodeData.credential ?? '', options)
+    const apiKey = getCredentialParam('apiKey', credentialData, nodeData)
+    const clientId = getCredentialParam('clientId', credentialData, nodeData)
+
+    let obj: MotorheadMemoryInput = {
+        returnMessages: true,
+        sessionId: sessionId ? sessionId : chatId,
+        memoryKey
+    }
+
+    if (baseURL) {
+        obj = {
+            ...obj,
+            url: baseURL
+        }
+    } else {
+        obj = {
+            ...obj,
+            apiKey,
+            clientId
+        }
+    }
+
+    return new MotorheadMemory(obj)
+}
+
 module.exports = { nodeClass: MotorMemory_Memory }

packages/components/nodes/memory/RedisBackedChatMemory/RedisBackedChatMemory.ts

@@ -58,31 +58,40 @@ class RedisBackedChatMemory_Memory implements INode {
     }
 
     async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
-        const baseURL = nodeData.inputs?.baseURL as string
-        const sessionId = nodeData.inputs?.sessionId as string
-        const sessionTTL = nodeData.inputs?.sessionTTL as number
-        const memoryKey = nodeData.inputs?.memoryKey as string
+        return initalizeRedis(nodeData, options)
+    }
 
-        const chatId = options?.chatId as string
-
-        const redisClient = createClient({ url: baseURL })
-        let obj: RedisChatMessageHistoryInput = {
-            sessionId: sessionId ? sessionId : chatId,
-            client: redisClient
-        }
-
-        if (sessionTTL) {
-            obj = {
-                ...obj,
-                sessionTTL
-            }
-        }
-
-        let redisChatMessageHistory = new RedisChatMessageHistory(obj)
-        let redis = new BufferMemory({ memoryKey, chatHistory: redisChatMessageHistory, returnMessages: true })
-
-        return redis
+    async clearSessionMemory(nodeData: INodeData, options: ICommonObject): Promise<void> {
+        const redis = initalizeRedis(nodeData, options)
+        redis.clear()
     }
 }
 
+const initalizeRedis = (nodeData: INodeData, options: ICommonObject): BufferMemory => {
+    const baseURL = nodeData.inputs?.baseURL as string
+    const sessionId = nodeData.inputs?.sessionId as string
+    const sessionTTL = nodeData.inputs?.sessionTTL as number
+    const memoryKey = nodeData.inputs?.memoryKey as string
+
+    const chatId = options?.chatId as string
+
+    const redisClient = createClient({ url: baseURL })
+    let obj: RedisChatMessageHistoryInput = {
+        sessionId: sessionId ? sessionId : chatId,
+        client: redisClient
+    }
+
+    if (sessionTTL) {
+        obj = {
+            ...obj,
+            sessionTTL
+        }
+    }
+
+    let redisChatMessageHistory = new RedisChatMessageHistory(obj)
+    let redis = new BufferMemory({ memoryKey, chatHistory: redisChatMessageHistory, returnMessages: true })
+
+    return redis
+}
+
 module.exports = { nodeClass: RedisBackedChatMemory_Memory }

packages/components/nodes/memory/ZepMemory/ZepMemory.ts

@@ -107,33 +107,12 @@ class ZepMemory_Memory implements INode {
     }
 
     async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
-        const baseURL = nodeData.inputs?.baseURL as string
-        const aiPrefix = nodeData.inputs?.aiPrefix as string
-        const humanPrefix = nodeData.inputs?.humanPrefix as string
-        const memoryKey = nodeData.inputs?.memoryKey as string
-        const inputKey = nodeData.inputs?.inputKey as string
         const autoSummaryTemplate = nodeData.inputs?.autoSummaryTemplate as string
         const autoSummary = nodeData.inputs?.autoSummary as boolean
-        const sessionId = nodeData.inputs?.sessionId as string
+
         const k = nodeData.inputs?.k as string
 
-        const chatId = options?.chatId as string
-
-        const credentialData = await getCredentialData(nodeData.credential ?? '', options)
-        const apiKey = getCredentialParam('apiKey', credentialData, nodeData)
-
-        const obj: ZepMemoryInput = {
-            baseURL,
-            sessionId: sessionId ? sessionId : chatId,
-            aiPrefix,
-            humanPrefix,
-            returnMessages: true,
-            memoryKey,
-            inputKey
-        }
-        if (apiKey) obj.apiKey = apiKey
-
-        let zep = new ZepMemory(obj)
+        let zep = await initalizeZep(nodeData, options)
 
         // hack to support summary
         let tmpFunc = zep.loadMemoryVariables
@@ -158,6 +137,38 @@ class ZepMemory_Memory implements INode {
         }
         return zep
     }
+
+    async clearSessionMemory(nodeData: INodeData, options: ICommonObject): Promise<void> {
+        const zep = await initalizeZep(nodeData, options)
+        await zep.clear()
+    }
+}
+
+const initalizeZep = async (nodeData: INodeData, options: ICommonObject): Promise<ZepMemory> => {
+    const baseURL = nodeData.inputs?.baseURL as string
+    const aiPrefix = nodeData.inputs?.aiPrefix as string
+    const humanPrefix = nodeData.inputs?.humanPrefix as string
+    const memoryKey = nodeData.inputs?.memoryKey as string
+    const inputKey = nodeData.inputs?.inputKey as string
+    const sessionId = nodeData.inputs?.sessionId as string
+
+    const chatId = options?.chatId as string
+
+    const credentialData = await getCredentialData(nodeData.credential ?? '', options)
+    const apiKey = getCredentialParam('apiKey', credentialData, nodeData)
+
+    const obj: ZepMemoryInput = {
+        baseURL,
+        sessionId: sessionId ? sessionId : chatId,
+        aiPrefix,
+        humanPrefix,
+        returnMessages: true,
+        memoryKey,
+        inputKey
+    }
+    if (apiKey) obj.apiKey = apiKey
+
+    return new ZepMemory(obj)
 }
 
 module.exports = { nodeClass: ZepMemory_Memory }

packages/components/nodes/prompts/ChatPromptTemplate/ChatPromptTemplate.ts

@@ -58,7 +58,7 @@ class ChatPromptTemplate_Prompts implements INode {
 
         let promptValues: ICommonObject = {}
         if (promptValuesStr) {
-            promptValues = JSON.parse(promptValuesStr.replace(/\s/g, ''))
+            promptValues = JSON.parse(promptValuesStr)
         }
         // @ts-ignore
         prompt.promptValues = promptValues

packages/components/nodes/prompts/FewShotPromptTemplate/FewShotPromptTemplate.ts

@@ -86,7 +86,7 @@ class FewShotPromptTemplate_Prompts implements INode {
         const examplePrompt = nodeData.inputs?.examplePrompt as PromptTemplate
 
         const inputVariables = getInputVariables(suffix)
-        const examples: Example[] = JSON.parse(examplesStr.replace(/\s/g, ''))
+        const examples: Example[] = JSON.parse(examplesStr)
 
         try {
             const obj: FewShotPromptTemplateInput = {

packages/components/nodes/prompts/PromptTemplate/PromptTemplate.ts

@@ -45,7 +45,7 @@ class PromptTemplate_Prompts implements INode {
 
         let promptValues: ICommonObject = {}
         if (promptValuesStr) {
-            promptValues = JSON.parse(promptValuesStr.replace(/\s/g, ''))
+            promptValues = JSON.parse(promptValuesStr)
         }
 
         const inputVariables = getInputVariables(template)

packages/components/nodes/retrievers/HydeRetriever/HydeRetriever.ts

@@ -0,0 +1,121 @@
+import { VectorStore } from 'langchain/vectorstores/base'
+import { INode, INodeData, INodeParams } from '../../../src/Interface'
+import { HydeRetriever, HydeRetrieverOptions, PromptKey } from 'langchain/retrievers/hyde'
+import { BaseLanguageModel } from 'langchain/base_language'
+import { PromptTemplate } from 'langchain/prompts'
+
+class HydeRetriever_Retrievers implements INode {
+    label: string
+    name: string
+    description: string
+    type: string
+    icon: string
+    category: string
+    baseClasses: string[]
+    inputs: INodeParams[]
+
+    constructor() {
+        this.label = 'Hyde Retriever'
+        this.name = 'HydeRetriever'
+        this.type = 'HydeRetriever'
+        this.icon = 'hyderetriever.svg'
+        this.category = 'Retrievers'
+        this.description = 'Use HyDE retriever to retrieve from a vector store'
+        this.baseClasses = [this.type, 'BaseRetriever']
+        this.inputs = [
+            {
+                label: 'Language Model',
+                name: 'model',
+                type: 'BaseLanguageModel'
+            },
+            {
+                label: 'Vector Store',
+                name: 'vectorStore',
+                type: 'VectorStore'
+            },
+            {
+                label: 'Prompt Key',
+                name: 'promptKey',
+                type: 'options',
+                options: [
+                    {
+                        label: 'websearch',
+                        name: 'websearch'
+                    },
+                    {
+                        label: 'scifact',
+                        name: 'scifact'
+                    },
+                    {
+                        label: 'arguana',
+                        name: 'arguana'
+                    },
+                    {
+                        label: 'trec-covid',
+                        name: 'trec-covid'
+                    },
+                    {
+                        label: 'fiqa',
+                        name: 'fiqa'
+                    },
+                    {
+                        label: 'dbpedia-entity',
+                        name: 'dbpedia-entity'
+                    },
+                    {
+                        label: 'trec-news',
+                        name: 'trec-news'
+                    },
+                    {
+                        label: 'mr-tydi',
+                        name: 'mr-tydi'
+                    }
+                ],
+                default: 'websearch'
+            },
+            {
+                label: 'Custom Prompt',
+                name: 'customPrompt',
+                description: 'If custom prompt is used, this will override Prompt Key',
+                placeholder: 'Please write a passage to answer the question\nQuestion: {question}\nPassage:',
+                type: 'string',
+                rows: 4,
+                additionalParams: true,
+                optional: true
+            },
+            {
+                label: 'Top K',
+                name: 'topK',
+                description: 'Number of top results to fetch. Default to 4',
+                placeholder: '4',
+                type: 'number',
+                default: 4,
+                additionalParams: true,
+                optional: true
+            }
+        ]
+    }
+
+    async init(nodeData: INodeData): Promise<any> {
+        const llm = nodeData.inputs?.model as BaseLanguageModel
+        const vectorStore = nodeData.inputs?.vectorStore as VectorStore
+        const promptKey = nodeData.inputs?.promptKey as PromptKey
+        const customPrompt = nodeData.inputs?.customPrompt as string
+        const topK = nodeData.inputs?.topK as string
+        const k = topK ? parseInt(topK, 10) : 4
+
+        const obj: HydeRetrieverOptions<any> = {
+            llm,
+            vectorStore,
+            k
+        }
+
+        if (customPrompt) obj.promptTemplate = PromptTemplate.fromTemplate(customPrompt)
+        else if (promptKey) obj.promptTemplate = promptKey
+
+        const retriever = new HydeRetriever(obj)
+        return retriever
+    }
+}
+
+module.exports = { nodeClass: HydeRetriever_Retrievers }

packages/components/nodes/retrievers/HydeRetriever/hyderetriever.svg

@@ -0,0 +1,9 @@
+<svg xmlns="http://www.w3.org/2000/svg" class="icon icon-tabler icon-tabler-database-export" width="24" height="24" viewBox="0 0 24 24" stroke-width="2" stroke="currentColor" fill="none" stroke-linecap="round" stroke-linejoin="round">
+   <path stroke="none" d="M0 0h24v24H0z" fill="none"></path>
+   <path d="M4 6c0 1.657 3.582 3 8 3s8 -1.343 8 -3s-3.582 -3 -8 -3s-8 1.343 -8 3"></path>
+   <path d="M4 6v6c0 1.657 3.582 3 8 3c1.118 0 2.183 -.086 3.15 -.241"></path>
+   <path d="M20 12v-6"></path>
+   <path d="M4 12v6c0 1.657 3.582 3 8 3c.157 0 .312 -.002 .466 -.005"></path>
+   <path d="M16 19h6"></path>
+   <path d="M19 16l3 3l-3 3"></path>
+</svg>

packages/components/nodes/textsplitters/HtmlToMarkdownTextSplitter/HtmlToMarkdownTextSplitter.ts

@@ -0,0 +1,70 @@
+import { INode, INodeData, INodeParams } from '../../../src/Interface'
+import { getBaseClasses } from '../../../src/utils'
+import { MarkdownTextSplitter, MarkdownTextSplitterParams } from 'langchain/text_splitter'
+import { NodeHtmlMarkdown } from 'node-html-markdown'
+
+class HtmlToMarkdownTextSplitter_TextSplitters implements INode {
+    label: string
+    name: string
+    description: string
+    type: string
+    icon: string
+    category: string
+    baseClasses: string[]
+    inputs: INodeParams[]
+
+    constructor() {
+        this.label = 'HtmlToMarkdown Text Splitter'
+        this.name = 'htmlToMarkdownTextSplitter'
+        this.type = 'HtmlToMarkdownTextSplitter'
+        this.icon = 'htmlToMarkdownTextSplitter.svg'
+        this.category = 'Text Splitters'
+        this.description = `Converts Html to Markdown and then split your content into documents based on the Markdown headers`
+        this.baseClasses = [this.type, ...getBaseClasses(HtmlToMarkdownTextSplitter)]
+        this.inputs = [
+            {
+                label: 'Chunk Size',
+                name: 'chunkSize',
+                type: 'number',
+                default: 1000,
+                optional: true
+            },
+            {
+                label: 'Chunk Overlap',
+                name: 'chunkOverlap',
+                type: 'number',
+                optional: true
+            }
+        ]
+    }
+
+    async init(nodeData: INodeData): Promise<any> {
+        const chunkSize = nodeData.inputs?.chunkSize as string
+        const chunkOverlap = nodeData.inputs?.chunkOverlap as string
+
+        const obj = {} as MarkdownTextSplitterParams
+
+        if (chunkSize) obj.chunkSize = parseInt(chunkSize, 10)
+        if (chunkOverlap) obj.chunkOverlap = parseInt(chunkOverlap, 10)
+
+        const splitter = new HtmlToMarkdownTextSplitter(obj)
+
+        return splitter
+    }
+}
+class HtmlToMarkdownTextSplitter extends MarkdownTextSplitter implements MarkdownTextSplitterParams {
+    constructor(fields?: Partial<MarkdownTextSplitterParams>) {
+        {
+            super(fields)
+        }
+    }
+    splitText(text: string): Promise<string[]> {
+        return new Promise((resolve) => {
+            const markdown = NodeHtmlMarkdown.translate(text)
+            super.splitText(markdown).then((result) => {
+                resolve(result)
+            })
+        })
+    }
+}
+module.exports = { nodeClass: HtmlToMarkdownTextSplitter_TextSplitters }

packages/components/nodes/textsplitters/HtmlToMarkdownTextSplitter/htmlToMarkdownTextSplitter.svg

@@ -0,0 +1,6 @@
+<svg xmlns="http://www.w3.org/2000/svg" class="icon icon-tabler icon-tabler-markdown" width="24" height="24" viewBox="0 0 24 24" stroke-width="2" stroke="currentColor" fill="none" stroke-linecap="round" stroke-linejoin="round">
+   <path stroke="none" d="M0 0h24v24H0z" fill="none"></path>
+   <path d="M3 5m0 2a2 2 0 0 1 2 -2h14a2 2 0 0 1 2 2v10a2 2 0 0 1 -2 2h-14a2 2 0 0 1 -2 -2z"></path>
+   <path d="M7 15v-6l2 2l2 -2v6"></path>
+   <path d="M14 13l2 2l2 -2m-2 2v-6"></path>
+</svg>

packages/components/nodes/tools/BraveSearchAPI/BraveSearchAPI.ts

@@ -0,0 +1,40 @@
+import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
+import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
+import { BraveSearch } from 'langchain/tools'
+
+class BraveSearchAPI_Tools implements INode {
+    label: string
+    name: string
+    description: string
+    type: string
+    icon: string
+    category: string
+    baseClasses: string[]
+    credential: INodeParams
+    inputs: INodeParams[]
+
+    constructor() {
+        this.label = 'BraveSearch API'
+        this.name = 'braveSearchAPI'
+        this.type = 'BraveSearchAPI'
+        this.icon = 'brave.svg'
+        this.category = 'Tools'
+        this.description = 'Wrapper around BraveSearch API - a real-time API to access Brave search results'
+        this.inputs = []
+        this.credential = {
+            label: 'Connect Credential',
+            name: 'credential',
+            type: 'credential',
+            credentialNames: ['braveSearchApi']
+        }
+        this.baseClasses = [this.type, ...getBaseClasses(BraveSearch)]
+    }
+
+    async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
+        const credentialData = await getCredentialData(nodeData.credential ?? '', options)
+        const braveApiKey = getCredentialParam('braveApiKey', credentialData, nodeData)
+        return new BraveSearch({ apiKey: braveApiKey })
+    }
+}
+
+module.exports = { nodeClass: BraveSearchAPI_Tools }

packages/components/nodes/tools/BraveSearchAPI/brave.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg"  viewBox="0 0 48 48" width="96px" height="96px"><path fill="#ff651f" d="M41,13l1,4l-4.09,16.35c-0.59,2.35-2.01,4.41-4.01,5.79l-8.19,5.68c-0.51,0.36-1.11,0.53-1.71,0.53	c-0.6,0-1.2-0.17-1.71-0.53l-8.19-5.68c-2-1.38-3.42-3.44-4.01-5.79L6,17l1-4l-1-2l3.25-3.25c1.05-1.05,2.6-1.44,4.02-0.99	c0.04,0.01,0.07,0.02,0.1,0.03L14,7l4-4h12l4,4l0.65-0.22c0.83-0.28,1.7-0.27,2.5,0c0.58,0.19,1.13,0.51,1.58,0.95	c0.01,0.01,0.01,0.01,0.02,0.02L42,11L41,13z"/><path fill="#f4592b" d="M38.73,7.73L33,11l-9,2l-9-3l-2.07-2.07c-0.56-0.56-1.41-0.74-2.15-0.44L8.67,8.33l0.58-0.58	c1.05-1.05,2.6-1.44,4.02-0.99c0.04,0.01,0.07,0.02,0.1,0.03L14,7l4-4h12l4,4l0.65-0.22c0.83-0.28,1.7-0.27,2.5,0	C37.73,6.97,38.28,7.29,38.73,7.73z"/><path fill="#fff" d="M32.51,23.49c-0.3,0.3-0.38,0.77-0.19,1.15l0.34,0.68c0.22,0.45,0.34,0.94,0.34,1.44	c0,0.8-0.29,1.57-0.83,2.16l-0.66,0.74c-0.32,0.21-0.72,0.23-1.04,0.05l-5.23-2.88c-0.59-0.4-0.6-1.27-0.01-1.66l3.91-2.66	c0.48-0.28,0.63-0.89,0.35-1.37l-1.9-3.16C27.28,17.46,27.45,17.24,28,17l6-3h-5l-3,0.75c-0.55,0.14-0.87,0.7-0.72,1.24l1.46,5.09	c0.14,0.51-0.14,1.05-0.65,1.22l-1.47,0.49c-0.21,0.07-0.41,0.11-0.62,0.11c-0.21,0-0.42-0.04-0.63-0.11l-1.46-0.49	c-0.51-0.17-0.79-0.71-0.65-1.22l1.46-5.09c0.15-0.54-0.17-1.1-0.72-1.24L19,14h-5l6,3c0.55,0.24,0.72,0.46,0.41,0.98l-1.9,3.16	c-0.28,0.48-0.13,1.09,0.35,1.37l3.91,2.66c0.59,0.39,0.58,1.26-0.01,1.66l-5.23,2.88c-0.32,0.18-0.72,0.16-1.04-0.05l-0.66-0.74	C15.29,28.33,15,27.56,15,26.76c0-0.5,0.12-0.99,0.34-1.44l0.34-0.68c0.19-0.38,0.11-0.85-0.19-1.15l-4.09-4.83	c-0.83-0.99-0.94-2.41-0.26-3.51l3.4-5.54c0.27-0.36,0.75-0.49,1.17-0.33l2.62,1.05c0.48,0.19,0.99,0.29,1.49,0.29	c0.61,0,1.23-0.14,1.79-0.42c0.75-0.38,1.57-0.57,2.39-0.57s1.64,0.19,2.39,0.57c1.03,0.51,2.22,0.56,3.28,0.13l2.62-1.05	c0.42-0.16,0.9-0.03,1.17,0.33l3.4,5.54c0.68,1.1,0.57,2.52-0.26,3.51L32.51,23.49z"/><path fill="#fff" d="M29.51,32.49l-4.8,3.8c-0.19,0.19-0.45,0.29-0.71,0.29s-0.52-0.1-0.71-0.29l-4.8-3.8	c-0.24-0.24-0.17-0.65,0.13-0.8l4.93-2.47c0.14-0.07,0.29-0.1,0.45-0.1s0.31,0.03,0.45,0.1l4.93,2.47	C29.68,31.84,29.75,32.25,29.51,32.49z"/><path fill="#ed4d01" d="M41,13l1,4l-4.09,16.35c-0.59,2.35-2.01,4.41-4.01,5.79l-8.19,5.68c-0.51,0.36-1.11,0.53-1.71,0.53	V10.36L25,12h7v-2l5.15-3.22c0.59,0.19,1.15,0.52,1.6,0.97L42,11L41,13z"/><path fill="#f5f5f5" d="M32.51,23.49c-0.3,0.3-0.38,0.77-0.19,1.15l0.34,0.68c0.22,0.45,0.34,0.94,0.34,1.44	c0,0.8-0.29,1.57-0.83,2.16l-0.66,0.74c-0.32,0.21-0.72,0.23-1.04,0.05l-5.23-2.88c-0.59-0.4-0.6-1.27-0.01-1.66l3.91-2.66	c0.48-0.28,0.63-0.89,0.35-1.37l-1.9-3.16C27.28,17.46,27.45,17.24,28,17l6-3h-5l-3,0.75c-0.55,0.14-0.87,0.7-0.72,1.24l1.46,5.09	c0.14,0.51-0.14,1.05-0.65,1.22l-1.47,0.49c-0.21,0.07-0.41,0.11-0.62,0.11V9.63c0.82,0,1.64,0.19,2.39,0.57	c1.03,0.51,2.22,0.56,3.28,0.13l2.62-1.05c0.42-0.16,0.9-0.03,1.17,0.33l3.4,5.54c0.68,1.1,0.57,2.52-0.26,3.51L32.51,23.49z"/><path fill="#f5f5f5" d="M29.51,32.49l-4.8,3.8c-0.19,0.19-0.45,0.29-0.71,0.29v-7.46c0.16,0,0.31,0.03,0.45,0.1l4.93,2.47	C29.68,31.84,29.75,32.25,29.51,32.49z"/></svg>

packages/components/nodes/tools/CustomTool/core.ts

@@ -2,7 +2,37 @@ import { z } from 'zod'
 import { CallbackManagerForToolRun } from 'langchain/callbacks'
 import { StructuredTool, ToolParams } from 'langchain/tools'
 import { NodeVM } from 'vm2'
-import { availableDependencies } from '../../../src/utils'
+
+/*
+ * List of dependencies allowed to be import in vm2
+ */
+const availableDependencies = [
+    '@dqbd/tiktoken',
+    '@getzep/zep-js',
+    '@huggingface/inference',
+    '@pinecone-database/pinecone',
+    '@supabase/supabase-js',
+    'axios',
+    'cheerio',
+    'chromadb',
+    'cohere-ai',
+    'd3-dsv',
+    'form-data',
+    'graphql',
+    'html-to-text',
+    'langchain',
+    'linkifyjs',
+    'mammoth',
+    'moment',
+    'node-fetch',
+    'pdf-parse',
+    'pdfjs-dist',
+    'playwright',
+    'puppeteer',
+    'srt-parser-2',
+    'typeorm',
+    'weaviate-ts-client'
+]
 
 export interface BaseDynamicToolInput extends ToolParams {
     name: string
@@ -51,25 +81,37 @@ export class DynamicStructuredTool<
             }
         }
 
+        const defaultAllowBuiltInDep = [
+            'assert',
+            'buffer',
+            'crypto',
+            'events',
+            'http',
+            'https',
+            'net',
+            'path',
+            'querystring',
+            'timers',
+            'tls',
+            'url',
+            'zlib'
+        ]
+
+        const builtinDeps = process.env.TOOL_FUNCTION_BUILTIN_DEP
+            ? defaultAllowBuiltInDep.concat(process.env.TOOL_FUNCTION_BUILTIN_DEP.split(','))
+            : defaultAllowBuiltInDep
+        const externalDeps = process.env.TOOL_FUNCTION_EXTERNAL_DEP ? process.env.TOOL_FUNCTION_EXTERNAL_DEP.split(',') : []
+        const deps = availableDependencies.concat(externalDeps)
+
         const options = {
             console: 'inherit',
             sandbox,
             require: {
-                external: false as boolean | { modules: string[] },
-                builtin: ['*']
+                external: { modules: deps },
+                builtin: builtinDeps
             }
         } as any
 
-        const external = JSON.stringify(availableDependencies)
-        if (external) {
-            const deps = JSON.parse(external)
-            if (deps && deps.length) {
-                options.require.external = {
-                    modules: deps
-                }
-            }
-        }
-
         const vm = new NodeVM(options)
         const response = await vm.run(`module.exports = async function() {${this.code}}()`, __dirname)
 

packages/components/nodes/vectorstores/Singlestore_Existing/Singlestore_Existing.ts

@@ -0,0 +1,144 @@
+import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
+import { Embeddings } from 'langchain/embeddings/base'
+import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
+import { SingleStoreVectorStore, SingleStoreVectorStoreConfig } from 'langchain/vectorstores/singlestore'
+
+class SingleStoreExisting_VectorStores implements INode {
+    label: string
+    name: string
+    description: string
+    type: string
+    icon: string
+    category: string
+    baseClasses: string[]
+    inputs: INodeParams[]
+    credential: INodeParams
+    outputs: INodeOutputsValue[]
+
+    constructor() {
+        this.label = 'SingleStore Load Existing Table'
+        this.name = 'singlestoreExisting'
+        this.type = 'SingleStore'
+        this.icon = 'singlestore.svg'
+        this.category = 'Vector Stores'
+        this.description = 'Load existing document from SingleStore'
+        this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
+        this.credential = {
+            label: 'Connect Credential',
+            name: 'credential',
+            type: 'credential',
+            description: 'Needed when using SingleStore cloud hosted',
+            optional: true,
+            credentialNames: ['singleStoreApi']
+        }
+        this.inputs = [
+            {
+                label: 'Embeddings',
+                name: 'embeddings',
+                type: 'Embeddings'
+            },
+            {
+                label: 'Host',
+                name: 'host',
+                type: 'string'
+            },
+            {
+                label: 'Database',
+                name: 'database',
+                type: 'string'
+            },
+            {
+                label: 'Table Name',
+                name: 'tableName',
+                type: 'string',
+                placeholder: 'embeddings',
+                additionalParams: true,
+                optional: true
+            },
+            {
+                label: 'Content Column Name',
+                name: 'contentColumnName',
+                type: 'string',
+                placeholder: 'content',
+                additionalParams: true,
+                optional: true
+            },
+            {
+                label: 'Vector Column Name',
+                name: 'vectorColumnName',
+                type: 'string',
+                placeholder: 'vector',
+                additionalParams: true,
+                optional: true
+            },
+            {
+                label: 'Metadata Column Name',
+                name: 'metadataColumnName',
+                type: 'string',
+                placeholder: 'metadata',
+                additionalParams: true,
+                optional: true
+            },
+            {
+                label: 'Top K',
+                name: 'topK',
+                placeholder: '4',
+                type: 'number',
+                additionalParams: true,
+                optional: true
+            }
+        ]
+        this.outputs = [
+            {
+                label: 'SingleStore Retriever',
+                name: 'retriever',
+                baseClasses: this.baseClasses
+            },
+            {
+                label: 'SingleStore Vector Store',
+                name: 'vectorStore',
+                baseClasses: [this.type, ...getBaseClasses(SingleStoreVectorStore)]
+            }
+        ]
+    }
+
+    async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
+        const credentialData = await getCredentialData(nodeData.credential ?? '', options)
+        const user = getCredentialParam('user', credentialData, nodeData)
+        const password = getCredentialParam('password', credentialData, nodeData)
+
+        const singleStoreConnectionConfig = {
+            connectionOptions: {
+                host: nodeData.inputs?.host as string,
+                port: 3306,
+                user,
+                password,
+                database: nodeData.inputs?.database as string
+            },
+            ...(nodeData.inputs?.tableName ? { tableName: nodeData.inputs.tableName as string } : {}),
+            ...(nodeData.inputs?.contentColumnName ? { contentColumnName: nodeData.inputs.contentColumnName as string } : {}),
+            ...(nodeData.inputs?.vectorColumnName ? { vectorColumnName: nodeData.inputs.vectorColumnName as string } : {}),
+            ...(nodeData.inputs?.metadataColumnName ? { metadataColumnName: nodeData.inputs.metadataColumnName as string } : {})
+        } as SingleStoreVectorStoreConfig
+
+        const embeddings = nodeData.inputs?.embeddings as Embeddings
+        const output = nodeData.outputs?.output as string
+        const topK = nodeData.inputs?.topK as string
+        const k = topK ? parseInt(topK, 10) : 4
+
+        let vectorStore: SingleStoreVectorStore
+
+        vectorStore = new SingleStoreVectorStore(embeddings, singleStoreConnectionConfig)
+
+        if (output === 'retriever') {
+            const retriever = vectorStore.asRetriever(k)
+            return retriever
+        } else if (output === 'vectorStore') {
+            ;(vectorStore as any).k = k
+            return vectorStore
+        }
+        return vectorStore
+    }
+}
+
+module.exports = { nodeClass: SingleStoreExisting_VectorStores }

packages/components/nodes/vectorstores/Singlestore_Existing/singlestore.svg

@@ -0,0 +1,20 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<svg width="256px" height="256px" viewBox="0 0 256 256" version="1.1" xmlns="http://www.w3.org/2000/svg" preserveAspectRatio="xMidYMid">
+    <title>SingleStore</title>
+    <defs>
+        <linearGradient x1="67.3449258%" y1="-26.0044686%" x2="-18.5227789%" y2="22.9877555%" id="singleStoreLinearGradient-1">
+            <stop stop-color="#FF7BFF" offset="0%"></stop>
+            <stop stop-color="#AA00FF" offset="35.0158%"></stop>
+            <stop stop-color="#8800CC" offset="100%"></stop>
+        </linearGradient>
+        <linearGradient x1="36.2591509%" y1="-19.3628763%" x2="111.72205%" y2="44.9975357%" id="singleStoreLinearGradient-2">
+            <stop stop-color="#FF7BFF" offset="3.54358%"></stop>
+            <stop stop-color="#8800CC" offset="57.6537%"></stop>
+            <stop stop-color="#311B92" offset="100%"></stop>
+        </linearGradient>
+    </defs>
+    <g>
+        <path d="M133.793438,0 C161.220114,7.62806846 186.208847,26.8506986 196.569923,50.3452712 C212.416431,88.4856134 208.759637,136.695058 191.389506,165.376569 C176.761849,188.8709 154.211058,201.381085 128.308006,201.075829 C88.0823106,200.770814 55.4752171,168.732936 55.1704441,128.456768 C55.1704441,88.1803574 86.8634599,54.9221798 128.308006,54.9221798 C135.012288,54.9221798 144.679052,55.851955 155.649674,60.4286222 C155.649674,60.4286222 147.762766,55.757287 127.50695,52.6192355 C69.3015772,44.9912153 0.621898574,89.095884 16.4683968,190.701711 C38.409617,229.757339 80.4639504,256.303989 128.308006,255.997284 C198.703093,255.692994 256.299161,198.024717 255.996071,127.236226 C255.996071,59.498847 200.836263,1.83073691 133.793438,0 Z" fill="url(#singleStoreLinearGradient-1)"></path>
+        <path d="M181.635561,54.0037552 C171.884031,33.5605356 151.771183,17.3889203 127.087223,10.9813448 C121.601791,9.45574074 115.811828,8.84547014 109.412318,8.540359 C99.9653199,8.540359 90.8230945,9.76087603 81.376096,12.2018618 C57.9109865,19.2196838 41.455174,32.950265 31.7034025,43.6293966 C19.20906,57.9701518 10.9810571,72.9211776 6.1052197,87.8722034 C6.1052197,88.1774594 5.8004708,88.4824739 5.8004708,89.0927445 C5.4957219,90.3132857 4.27677462,93.9746678 4.27677462,94.8901944 C3.97202572,95.500465 3.97202572,96.4157502 3.66730098,97.0260207 C3.36255208,98.2465619 3.05780318,99.4668616 2.75307844,100.687403 C2.75307844,100.992659 2.75307844,101.297673 2.44832954,101.602688 C-5.47492441,140.963571 7.68750379,176.286091 15.6107577,189.406305 C17.5925312,192.688049 19.2199033,195.425935 20.8508738,197.938019 C2.87119611,100.298588 54.558966,53.6984992 113.373885,52.477958 C144.152582,51.8679289 174.931278,64.3778723 189.863709,89.3980006 C188.949389,75.6672745 188.035312,68.0392543 181.635561,54.0037552 Z" fill="url(#singleStoreLinearGradient-2)"></path>
+    </g>
+</svg>

packages/components/nodes/vectorstores/Singlestore_Upsert/Singlestore_Upsert.ts

@@ -0,0 +1,160 @@
+import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
+import { Embeddings } from 'langchain/embeddings/base'
+import { Document } from 'langchain/document'
+import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
+import { SingleStoreVectorStore, SingleStoreVectorStoreConfig } from 'langchain/vectorstores/singlestore'
+import { flatten } from 'lodash'
+
+class SingleStoreUpsert_VectorStores implements INode {
+    label: string
+    name: string
+    description: string
+    type: string
+    icon: string
+    category: string
+    baseClasses: string[]
+    inputs: INodeParams[]
+    credential: INodeParams
+    outputs: INodeOutputsValue[]
+
+    constructor() {
+        this.label = 'SingleStore Upsert Document'
+        this.name = 'singlestoreUpsert'
+        this.type = 'SingleStore'
+        this.icon = 'singlestore.svg'
+        this.category = 'Vector Stores'
+        this.description = 'Upsert documents to SingleStore'
+        this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
+        this.credential = {
+            label: 'Connect Credential',
+            name: 'credential',
+            type: 'credential',
+            description: 'Needed when using SingleStore cloud hosted',
+            optional: true,
+            credentialNames: ['singleStoreApi']
+        }
+        this.inputs = [
+            {
+                label: 'Document',
+                name: 'document',
+                type: 'Document',
+                list: true
+            },
+            {
+                label: 'Embeddings',
+                name: 'embeddings',
+                type: 'Embeddings'
+            },
+            {
+                label: 'Host',
+                name: 'host',
+                type: 'string'
+            },
+            {
+                label: 'Database',
+                name: 'database',
+                type: 'string'
+            },
+            {
+                label: 'Table Name',
+                name: 'tableName',
+                type: 'string',
+                placeholder: 'embeddings',
+                additionalParams: true,
+                optional: true
+            },
+            {
+                label: 'Content Column Name',
+                name: 'contentColumnName',
+                type: 'string',
+                placeholder: 'content',
+                additionalParams: true,
+                optional: true
+            },
+            {
+                label: 'Vector Column Name',
+                name: 'vectorColumnName',
+                type: 'string',
+                placeholder: 'vector',
+                additionalParams: true,
+                optional: true
+            },
+            {
+                label: 'Metadata Column Name',
+                name: 'metadataColumnName',
+                type: 'string',
+                placeholder: 'metadata',
+                additionalParams: true,
+                optional: true
+            },
+            {
+                label: 'Top K',
+                name: 'topK',
+                placeholder: '4',
+                type: 'number',
+                additionalParams: true,
+                optional: true
+            }
+        ]
+        this.outputs = [
+            {
+                label: 'SingleStore Retriever',
+                name: 'retriever',
+                baseClasses: this.baseClasses
+            },
+            {
+                label: 'SingleStore Vector Store',
+                name: 'vectorStore',
+                baseClasses: [this.type, ...getBaseClasses(SingleStoreVectorStore)]
+            }
+        ]
+    }
+
+    async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
+        const credentialData = await getCredentialData(nodeData.credential ?? '', options)
+        const user = getCredentialParam('user', credentialData, nodeData)
+        const password = getCredentialParam('password', credentialData, nodeData)
+
+        const singleStoreConnectionConfig = {
+            connectionOptions: {
+                host: nodeData.inputs?.host as string,
+                port: 3306,
+                user,
+                password,
+                database: nodeData.inputs?.database as string
+            },
+            ...(nodeData.inputs?.tableName ? { tableName: nodeData.inputs.tableName as string } : {}),
+            ...(nodeData.inputs?.contentColumnName ? { contentColumnName: nodeData.inputs.contentColumnName as string } : {}),
+            ...(nodeData.inputs?.vectorColumnName ? { vectorColumnName: nodeData.inputs.vectorColumnName as string } : {}),
+            ...(nodeData.inputs?.metadataColumnName ? { metadataColumnName: nodeData.inputs.metadataColumnName as string } : {})
+        } as SingleStoreVectorStoreConfig
+
+        const docs = nodeData.inputs?.document as Document[]
+        const embeddings = nodeData.inputs?.embeddings as Embeddings
+        const output = nodeData.outputs?.output as string
+        const topK = nodeData.inputs?.topK as string
+        const k = topK ? parseInt(topK, 10) : 4
+
+        const flattenDocs = docs && docs.length ? flatten(docs) : []
+        const finalDocs = []
+        for (let i = 0; i < flattenDocs.length; i += 1) {
+            finalDocs.push(new Document(flattenDocs[i]))
+        }
+
+        let vectorStore: SingleStoreVectorStore
+
+        vectorStore = new SingleStoreVectorStore(embeddings, singleStoreConnectionConfig)
+        vectorStore.addDocuments.bind(vectorStore)(finalDocs)
+
+        if (output === 'retriever') {
+            const retriever = vectorStore.asRetriever(k)
+            return retriever
+        } else if (output === 'vectorStore') {
+            ;(vectorStore as any).k = k
+            return vectorStore
+        }
+        return vectorStore
+    }
+}
+
+module.exports = { nodeClass: SingleStoreUpsert_VectorStores }

packages/components/nodes/vectorstores/Singlestore_Upsert/singlestore.svg

@@ -0,0 +1,20 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<svg width="256px" height="256px" viewBox="0 0 256 256" version="1.1" xmlns="http://www.w3.org/2000/svg" preserveAspectRatio="xMidYMid">
+    <title>SingleStore</title>
+    <defs>
+        <linearGradient x1="67.3449258%" y1="-26.0044686%" x2="-18.5227789%" y2="22.9877555%" id="singleStoreLinearGradient-1">
+            <stop stop-color="#FF7BFF" offset="0%"></stop>
+            <stop stop-color="#AA00FF" offset="35.0158%"></stop>
+            <stop stop-color="#8800CC" offset="100%"></stop>
+        </linearGradient>
+        <linearGradient x1="36.2591509%" y1="-19.3628763%" x2="111.72205%" y2="44.9975357%" id="singleStoreLinearGradient-2">
+            <stop stop-color="#FF7BFF" offset="3.54358%"></stop>
+            <stop stop-color="#8800CC" offset="57.6537%"></stop>
+            <stop stop-color="#311B92" offset="100%"></stop>
+        </linearGradient>
+    </defs>
+    <g>
+        <path d="M133.793438,0 C161.220114,7.62806846 186.208847,26.8506986 196.569923,50.3452712 C212.416431,88.4856134 208.759637,136.695058 191.389506,165.376569 C176.761849,188.8709 154.211058,201.381085 128.308006,201.075829 C88.0823106,200.770814 55.4752171,168.732936 55.1704441,128.456768 C55.1704441,88.1803574 86.8634599,54.9221798 128.308006,54.9221798 C135.012288,54.9221798 144.679052,55.851955 155.649674,60.4286222 C155.649674,60.4286222 147.762766,55.757287 127.50695,52.6192355 C69.3015772,44.9912153 0.621898574,89.095884 16.4683968,190.701711 C38.409617,229.757339 80.4639504,256.303989 128.308006,255.997284 C198.703093,255.692994 256.299161,198.024717 255.996071,127.236226 C255.996071,59.498847 200.836263,1.83073691 133.793438,0 Z" fill="url(#singleStoreLinearGradient-1)"></path>
+        <path d="M181.635561,54.0037552 C171.884031,33.5605356 151.771183,17.3889203 127.087223,10.9813448 C121.601791,9.45574074 115.811828,8.84547014 109.412318,8.540359 C99.9653199,8.540359 90.8230945,9.76087603 81.376096,12.2018618 C57.9109865,19.2196838 41.455174,32.950265 31.7034025,43.6293966 C19.20906,57.9701518 10.9810571,72.9211776 6.1052197,87.8722034 C6.1052197,88.1774594 5.8004708,88.4824739 5.8004708,89.0927445 C5.4957219,90.3132857 4.27677462,93.9746678 4.27677462,94.8901944 C3.97202572,95.500465 3.97202572,96.4157502 3.66730098,97.0260207 C3.36255208,98.2465619 3.05780318,99.4668616 2.75307844,100.687403 C2.75307844,100.992659 2.75307844,101.297673 2.44832954,101.602688 C-5.47492441,140.963571 7.68750379,176.286091 15.6107577,189.406305 C17.5925312,192.688049 19.2199033,195.425935 20.8508738,197.938019 C2.87119611,100.298588 54.558966,53.6984992 113.373885,52.477958 C144.152582,51.8679289 174.931278,64.3778723 189.863709,89.3980006 C188.949389,75.6672745 188.035312,68.0392543 181.635561,54.0037552 Z" fill="url(#singleStoreLinearGradient-2)"></path>
+    </g>
+</svg>