Merge branch 'main' into feature/Credential

# Conflicts: # README.md # docker/.env.example # packages/components/nodes/documentloaders/Notion/NotionDB.ts # packages/components/nodes/memory/DynamoDb/DynamoDb.ts # packages/components/nodes/memory/MotorheadMemory/MotorheadMemory.ts # packages/components/nodes/memory/ZepMemory/ZepMemory.ts # packages/components/package.json # packages/components/src/utils.ts # packages/server/.env.example # packages/server/README.md # packages/server/marketplaces/chatflows/Conversational Retrieval QA Chain.json # packages/server/src/ChildProcess.ts # packages/server/src/DataSource.ts # packages/server/src/commands/start.ts # packages/server/src/index.ts # packages/server/src/utils/index.ts # packages/server/src/utils/logger.ts
2026-06-28 21:00:58 +03:00 · 2023-07-27 11:26:34 +01:00
parent 0665dcef55 e2d3f27842
commit 61dabbb7da
107 changed files with 4347 additions and 826 deletions
@@ -0,0 +1,231 @@
+import { ICommonObject, INode, INodeData, INodeParams, PromptTemplate } from '../../../src/Interface'
+import { AgentExecutor } from 'langchain/agents'
+import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
+import { LoadPyodide, finalSystemPrompt, systemPrompt } from './core'
+import { LLMChain } from 'langchain/chains'
+import { BaseLanguageModel } from 'langchain/base_language'
+import { ConsoleCallbackHandler, CustomChainHandler } from '../../../src/handler'
+import axios from 'axios'
+
+class Airtable_Agents implements INode {
+    label: string
+    name: string
+    description: string
+    type: string
+    icon: string
+    category: string
+    baseClasses: string[]
+    credential: INodeParams
+    inputs: INodeParams[]
+
+    constructor() {
+        this.label = 'Airtable Agent'
+        this.name = 'airtableAgent'
+        this.type = 'AgentExecutor'
+        this.category = 'Agents'
+        this.icon = 'airtable.svg'
+        this.description = 'Agent used to to answer queries on Airtable table'
+        this.baseClasses = [this.type, ...getBaseClasses(AgentExecutor)]
+        this.credential = {
+            label: 'Connect Credential',
+            name: 'credential',
+            type: 'credential',
+            credentialNames: ['airtableApi']
+        }
+        this.inputs = [
+            {
+                label: 'Language Model',
+                name: 'model',
+                type: 'BaseLanguageModel'
+            },
+            {
+                label: 'Base Id',
+                name: 'baseId',
+                type: 'string',
+                placeholder: 'app11RobdGoX0YNsC',
+                description:
+                    'If your table URL looks like: https://airtable.com/app11RobdGoX0YNsC/tblJdmvbrgizbYICO/viw9UrP77Id0CE4ee, app11RovdGoX0YNsC is the base id'
+            },
+            {
+                label: 'Table Id',
+                name: 'tableId',
+                type: 'string',
+                placeholder: 'tblJdmvbrgizbYICO',
+                description:
+                    'If your table URL looks like: https://airtable.com/app11RobdGoX0YNsC/tblJdmvbrgizbYICO/viw9UrP77Id0CE4ee, tblJdmvbrgizbYICO is the table id'
+            },
+            {
+                label: 'Return All',
+                name: 'returnAll',
+                type: 'boolean',
+                default: true,
+                additionalParams: true,
+                description: 'If all results should be returned or only up to a given limit'
+            },
+            {
+                label: 'Limit',
+                name: 'limit',
+                type: 'number',
+                default: 100,
+                step: 1,
+                additionalParams: true,
+                description: 'Number of results to return'
+            }
+        ]
+    }
+
+    async init(): Promise<any> {
+        // Not used
+        return undefined
+    }
+
+    async run(nodeData: INodeData, input: string, options: ICommonObject): Promise<string> {
+        const model = nodeData.inputs?.model as BaseLanguageModel
+        const baseId = nodeData.inputs?.baseId as string
+        const tableId = nodeData.inputs?.tableId as string
+        const returnAll = nodeData.inputs?.returnAll as boolean
+        const limit = nodeData.inputs?.limit as string
+
+        const credentialData = await getCredentialData(nodeData.credential ?? '', options)
+        const accessToken = getCredentialParam('accessToken', credentialData, nodeData)
+
+        let airtableData: ICommonObject[] = []
+
+        if (returnAll) {
+            airtableData = await loadAll(baseId, tableId, accessToken)
+        } else {
+            airtableData = await loadLimit(limit ? parseInt(limit, 10) : 100, baseId, tableId, accessToken)
+        }
+
+        let base64String = Buffer.from(JSON.stringify(airtableData)).toString('base64')
+
+        const loggerHandler = new ConsoleCallbackHandler(options.logger)
+        const handler = new CustomChainHandler(options.socketIO, options.socketIOClientId)
+
+        const pyodide = await LoadPyodide()
+
+        // First load the csv file and get the dataframe dictionary of column types
+        // For example using titanic.csv: {'PassengerId': 'int64', 'Survived': 'int64', 'Pclass': 'int64', 'Name': 'object', 'Sex': 'object', 'Age': 'float64', 'SibSp': 'int64', 'Parch': 'int64', 'Ticket': 'object', 'Fare': 'float64', 'Cabin': 'object', 'Embarked': 'object'}
+        let dataframeColDict = ''
+        try {
+            const code = `import pandas as pd
+import base64
+import json
+
+base64_string = "${base64String}"
+
+decoded_data = base64.b64decode(base64_string)
+
+json_data = json.loads(decoded_data)
+
+df = pd.DataFrame(json_data)
+my_dict = df.dtypes.astype(str).to_dict()
+print(my_dict)
+json.dumps(my_dict)`
+            dataframeColDict = await pyodide.runPythonAsync(code)
+        } catch (error) {
+            throw new Error(error)
+        }
+
+        // Then tell GPT to come out with ONLY python code
+        // For example: len(df), df[df['SibSp'] > 3]['PassengerId'].count()
+        let pythonCode = ''
+        if (dataframeColDict) {
+            const chain = new LLMChain({
+                llm: model,
+                prompt: PromptTemplate.fromTemplate(systemPrompt),
+                verbose: process.env.DEBUG === 'true' ? true : false
+            })
+            const inputs = {
+                dict: dataframeColDict,
+                question: input
+            }
+            const res = await chain.call(inputs, [loggerHandler])
+            pythonCode = res?.text
+        }
+
+        // Then run the code using Pyodide
+        let finalResult = ''
+        if (pythonCode) {
+            try {
+                const code = `import pandas as pd\n${pythonCode}`
+                finalResult = await pyodide.runPythonAsync(code)
+            } catch (error) {
+                throw new Error(`Sorry, I'm unable to find answer for question: "${input}" using follwoing code: "${pythonCode}"`)
+            }
+        }
+
+        // Finally, return a complete answer
+        if (finalResult) {
+            const chain = new LLMChain({
+                llm: model,
+                prompt: PromptTemplate.fromTemplate(finalSystemPrompt),
+                verbose: process.env.DEBUG === 'true' ? true : false
+            })
+            const inputs = {
+                question: input,
+                answer: finalResult
+            }
+
+            if (options.socketIO && options.socketIOClientId) {
+                const result = await chain.call(inputs, [loggerHandler, handler])
+                return result?.text
+            } else {
+                const result = await chain.call(inputs, [loggerHandler])
+                return result?.text
+            }
+        }
+
+        return pythonCode
+    }
+}
+
+interface AirtableLoaderResponse {
+    records: AirtableLoaderPage[]
+    offset?: string
+}
+
+interface AirtableLoaderPage {
+    id: string
+    createdTime: string
+    fields: ICommonObject
+}
+
+const fetchAirtableData = async (url: string, params: ICommonObject, accessToken: string): Promise<AirtableLoaderResponse> => {
+    try {
+        const headers = {
+            Authorization: `Bearer ${accessToken}`,
+            'Content-Type': 'application/json',
+            Accept: 'application/json'
+        }
+        const response = await axios.get(url, { params, headers })
+        return response.data
+    } catch (error) {
+        throw new Error(`Failed to fetch ${url} from Airtable: ${error}`)
+    }
+}
+
+const loadAll = async (baseId: string, tableId: string, accessToken: string): Promise<ICommonObject[]> => {
+    const params: ICommonObject = { pageSize: 100 }
+    let data: AirtableLoaderResponse
+    let returnPages: AirtableLoaderPage[] = []
+
+    do {
+        data = await fetchAirtableData(`https://api.airtable.com/v0/${baseId}/${tableId}`, params, accessToken)
+        returnPages.push.apply(returnPages, data.records)
+        params.offset = data.offset
+    } while (data.offset !== undefined)
+
+    return data.records.map((page) => page.fields)
+}
+
+const loadLimit = async (limit: number, baseId: string, tableId: string, accessToken: string): Promise<ICommonObject[]> => {
+    const params = { maxRecords: limit }
+    const data = await fetchAirtableData(`https://api.airtable.com/v0/${baseId}/${tableId}`, params, accessToken)
+    if (data.records.length === 0) {
+        return []
+    }
+    return data.records.map((page) => page.fields)
+}
+
+module.exports = { nodeClass: Airtable_Agents }
@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<svg width="256px" height="215px" viewBox="0 0 256 215" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" preserveAspectRatio="xMidYMid">
+	<g>
+		<path d="M114.25873,2.70101695 L18.8604023,42.1756384 C13.5552723,44.3711638 13.6102328,51.9065311 18.9486282,54.0225085 L114.746142,92.0117514 C123.163769,95.3498757 132.537419,95.3498757 140.9536,92.0117514 L236.75256,54.0225085 C242.08951,51.9065311 242.145916,44.3711638 236.83934,42.1756384 L141.442459,2.70101695 C132.738459,-0.900338983 122.961284,-0.900338983 114.25873,2.70101695" fill="#FFBF00"></path>
+		<path d="M136.349071,112.756863 L136.349071,207.659101 C136.349071,212.173089 140.900664,215.263892 145.096461,213.600615 L251.844122,172.166219 C254.281184,171.200072 255.879376,168.845451 255.879376,166.224705 L255.879376,71.3224678 C255.879376,66.8084791 251.327783,63.7176768 247.131986,65.3809537 L140.384325,106.815349 C137.94871,107.781496 136.349071,110.136118 136.349071,112.756863" fill="#26B5F8"></path>
+		<path d="M111.422771,117.65355 L79.742409,132.949912 L76.5257763,134.504714 L9.65047684,166.548104 C5.4112904,168.593211 0.000578531073,165.503855 0.000578531073,160.794612 L0.000578531073,71.7210757 C0.000578531073,70.0173017 0.874160452,68.5463864 2.04568588,67.4384994 C2.53454463,66.9481944 3.08848814,66.5446689 3.66412655,66.2250305 C5.26231864,65.2661153 7.54173107,65.0101153 9.47981017,65.7766689 L110.890522,105.957098 C116.045234,108.002206 116.450206,115.225166 111.422771,117.65355" fill="#ED3049"></path>
+		<path d="M111.422771,117.65355 L79.742409,132.949912 L2.04568588,67.4384994 C2.53454463,66.9481944 3.08848814,66.5446689 3.66412655,66.2250305 C5.26231864,65.2661153 7.54173107,65.0101153 9.47981017,65.7766689 L110.890522,105.957098 C116.045234,108.002206 116.450206,115.225166 111.422771,117.65355" fill-opacity="0.25" fill="#000000"></path>
+	</g>
+</svg>
@@ -0,0 +1,29 @@
+import type { PyodideInterface } from 'pyodide'
+import * as path from 'path'
+import { getUserHome } from '../../../src/utils'
+
+let pyodideInstance: PyodideInterface | undefined
+
+export async function LoadPyodide(): Promise<PyodideInterface> {
+    if (pyodideInstance === undefined) {
+        const { loadPyodide } = await import('pyodide')
+        const obj: any = { packageCacheDir: path.join(getUserHome(), '.flowise', 'pyodideCacheDir') }
+        pyodideInstance = await loadPyodide(obj)
+        await pyodideInstance.loadPackage(['pandas', 'numpy'])
+    }
+
+    return pyodideInstance
+}
+
+export const systemPrompt = `You are working with a pandas dataframe in Python. The name of the dataframe is df.
+
+The columns and data types of a dataframe are given below as a Python dictionary with keys showing column names and values showing the data types.
+{dict}
+
+I will ask question, and you will output the Python code using pandas dataframe to answer my question. Do not provide any explanations. Do not respond with anything except the output of the code.
+
+Question: {question}
+Output Code:`
+
+export const finalSystemPrompt = `You are given the question: {question}. You have an answer to the question: {answer}. Rephrase the answer into a standalone answer.
+Standalone Answer:`