Merge branch 'main' into feature/Credential

# Conflicts: # README.md # docker/.env.example # packages/components/nodes/documentloaders/Notion/NotionDB.ts # packages/components/nodes/memory/DynamoDb/DynamoDb.ts # packages/components/nodes/memory/MotorheadMemory/MotorheadMemory.ts # packages/components/nodes/memory/ZepMemory/ZepMemory.ts # packages/components/package.json # packages/components/src/utils.ts # packages/server/.env.example # packages/server/README.md # packages/server/marketplaces/chatflows/Conversational Retrieval QA Chain.json # packages/server/src/ChildProcess.ts # packages/server/src/DataSource.ts # packages/server/src/commands/start.ts # packages/server/src/index.ts # packages/server/src/utils/index.ts # packages/server/src/utils/logger.ts
2026-06-28 15:00:57 +03:00 · 2023-07-27 11:26:34 +01:00
parent 0665dcef55 e2d3f27842
commit 61dabbb7da
107 changed files with 4347 additions and 826 deletions
@@ -0,0 +1,198 @@
+import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
+import { TextSplitter } from 'langchain/text_splitter'
+import { BaseDocumentLoader } from 'langchain/document_loaders/base'
+import { Document } from 'langchain/document'
+import axios, { AxiosRequestConfig } from 'axios'
+
+class API_DocumentLoaders implements INode {
+    label: string
+    name: string
+    description: string
+    type: string
+    icon: string
+    category: string
+    baseClasses: string[]
+    inputs?: INodeParams[]
+
+    constructor() {
+        this.label = 'API Loader'
+        this.name = 'apiLoader'
+        this.type = 'Document'
+        this.icon = 'api-loader.png'
+        this.category = 'Document Loaders'
+        this.description = `Load data from an API`
+        this.baseClasses = [this.type]
+        this.inputs = [
+            {
+                label: 'Text Splitter',
+                name: 'textSplitter',
+                type: 'TextSplitter',
+                optional: true
+            },
+            {
+                label: 'Method',
+                name: 'method',
+                type: 'options',
+                options: [
+                    {
+                        label: 'GET',
+                        name: 'GET'
+                    },
+                    {
+                        label: 'POST',
+                        name: 'POST'
+                    }
+                ]
+            },
+            {
+                label: 'URL',
+                name: 'url',
+                type: 'string'
+            },
+            {
+                label: 'Headers',
+                name: 'headers',
+                type: 'json',
+                additionalParams: true,
+                optional: true
+            },
+            {
+                label: 'Body',
+                name: 'body',
+                type: 'json',
+                description:
+                    'JSON body for the POST request. If not specified, agent will try to figure out itself from AIPlugin if provided',
+                additionalParams: true,
+                optional: true
+            }
+        ]
+    }
+    async init(nodeData: INodeData): Promise<any> {
+        const headers = nodeData.inputs?.headers as string
+        const url = nodeData.inputs?.url as string
+        const body = nodeData.inputs?.body as string
+        const method = nodeData.inputs?.method as string
+        const textSplitter = nodeData.inputs?.textSplitter as TextSplitter
+        const metadata = nodeData.inputs?.metadata
+
+        const options: ApiLoaderParams = {
+            url,
+            method
+        }
+
+        if (headers) {
+            const parsedHeaders = typeof headers === 'object' ? headers : JSON.parse(headers)
+            options.headers = parsedHeaders
+        }
+
+        if (body) {
+            const parsedBody = typeof body === 'object' ? body : JSON.parse(body)
+            options.body = parsedBody
+        }
+
+        const loader = new ApiLoader(options)
+
+        let docs = []
+
+        if (textSplitter) {
+            docs = await loader.loadAndSplit(textSplitter)
+        } else {
+            docs = await loader.load()
+        }
+
+        if (metadata) {
+            const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata)
+            let finaldocs = []
+            for (const doc of docs) {
+                const newdoc = {
+                    ...doc,
+                    metadata: {
+                        ...doc.metadata,
+                        ...parsedMetadata
+                    }
+                }
+                finaldocs.push(newdoc)
+            }
+            return finaldocs
+        }
+
+        return docs
+    }
+}
+
+interface ApiLoaderParams {
+    url: string
+    method: string
+    headers?: ICommonObject
+    body?: ICommonObject
+}
+
+class ApiLoader extends BaseDocumentLoader {
+    public readonly url: string
+
+    public readonly headers?: ICommonObject
+
+    public readonly body?: ICommonObject
+
+    public readonly method: string
+
+    constructor({ url, headers, body, method }: ApiLoaderParams) {
+        super()
+        this.url = url
+        this.headers = headers
+        this.body = body
+        this.method = method
+    }
+
+    public async load(): Promise<Document[]> {
+        if (this.method === 'POST') {
+            return this.executePostRequest(this.url, this.headers, this.body)
+        } else {
+            return this.executeGetRequest(this.url, this.headers)
+        }
+    }
+
+    protected async executeGetRequest(url: string, headers?: ICommonObject): Promise<Document[]> {
+        try {
+            const config: AxiosRequestConfig = {}
+            if (headers) {
+                config.headers = headers
+            }
+            const response = await axios.get(url, config)
+            const responseJsonString = JSON.stringify(response.data, null, 2)
+            const doc = new Document({
+                pageContent: responseJsonString,
+                metadata: {
+                    url
+                }
+            })
+            return [doc]
+        } catch (error) {
+            throw new Error(`Failed to fetch ${url}: ${error}`)
+        }
+    }
+
+    protected async executePostRequest(url: string, headers?: ICommonObject, body?: ICommonObject): Promise<Document[]> {
+        try {
+            const config: AxiosRequestConfig = {}
+            if (headers) {
+                config.headers = headers
+            }
+            const response = await axios.post(url, body ?? {}, config)
+            const responseJsonString = JSON.stringify(response.data, null, 2)
+            const doc = new Document({
+                pageContent: responseJsonString,
+                metadata: {
+                    url
+                }
+            })
+            return [doc]
+        } catch (error) {
+            throw new Error(`Failed to post ${url}: ${error}`)
+        }
+    }
+}
+
+module.exports = {
+    nodeClass: API_DocumentLoaders
+}
@@ -0,0 +1,229 @@
+import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
+import { TextSplitter } from 'langchain/text_splitter'
+import { BaseDocumentLoader } from 'langchain/document_loaders/base'
+import { Document } from 'langchain/document'
+import axios from 'axios'
+import { getCredentialData, getCredentialParam } from '../../../src/utils'
+
+class Airtable_DocumentLoaders implements INode {
+    label: string
+    name: string
+    description: string
+    type: string
+    icon: string
+    category: string
+    baseClasses: string[]
+    credential: INodeParams
+    inputs?: INodeParams[]
+
+    constructor() {
+        this.label = 'Airtable'
+        this.name = 'airtable'
+        this.type = 'Document'
+        this.icon = 'airtable.svg'
+        this.category = 'Document Loaders'
+        this.description = `Load data from Airtable table`
+        this.baseClasses = [this.type]
+        this.credential = {
+            label: 'Connect Credential',
+            name: 'credential',
+            type: 'credential',
+            credentialNames: ['airtableApi']
+        }
+        this.inputs = [
+            {
+                label: 'Text Splitter',
+                name: 'textSplitter',
+                type: 'TextSplitter',
+                optional: true
+            },
+            {
+                label: 'Base Id',
+                name: 'baseId',
+                type: 'string',
+                placeholder: 'app11RobdGoX0YNsC',
+                description:
+                    'If your table URL looks like: https://airtable.com/app11RobdGoX0YNsC/tblJdmvbrgizbYICO/viw9UrP77Id0CE4ee, app11RovdGoX0YNsC is the base id'
+            },
+            {
+                label: 'Table Id',
+                name: 'tableId',
+                type: 'string',
+                placeholder: 'tblJdmvbrgizbYICO',
+                description:
+                    'If your table URL looks like: https://airtable.com/app11RobdGoX0YNsC/tblJdmvbrgizbYICO/viw9UrP77Id0CE4ee, tblJdmvbrgizbYICO is the table id'
+            },
+            {
+                label: 'Return All',
+                name: 'returnAll',
+                type: 'boolean',
+                default: true,
+                additionalParams: true,
+                description: 'If all results should be returned or only up to a given limit'
+            },
+            {
+                label: 'Limit',
+                name: 'limit',
+                type: 'number',
+                default: 100,
+                step: 1,
+                additionalParams: true,
+                description: 'Number of results to return'
+            },
+            {
+                label: 'Metadata',
+                name: 'metadata',
+                type: 'json',
+                optional: true,
+                additionalParams: true
+            }
+        ]
+    }
+    async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
+        const baseId = nodeData.inputs?.baseId as string
+        const tableId = nodeData.inputs?.tableId as string
+        const returnAll = nodeData.inputs?.returnAll as boolean
+        const limit = nodeData.inputs?.limit as string
+        const textSplitter = nodeData.inputs?.textSplitter as TextSplitter
+        const metadata = nodeData.inputs?.metadata
+
+        const credentialData = await getCredentialData(nodeData.credential ?? '', options)
+        const accessToken = getCredentialParam('accessToken', credentialData, nodeData)
+
+        const airtableOptions: AirtableLoaderParams = {
+            baseId,
+            tableId,
+            returnAll,
+            accessToken,
+            limit: limit ? parseInt(limit, 10) : 100
+        }
+
+        const loader = new AirtableLoader(airtableOptions)
+
+        let docs = []
+
+        if (textSplitter) {
+            docs = await loader.loadAndSplit(textSplitter)
+        } else {
+            docs = await loader.load()
+        }
+
+        if (metadata) {
+            const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata)
+            let finaldocs = []
+            for (const doc of docs) {
+                const newdoc = {
+                    ...doc,
+                    metadata: {
+                        ...doc.metadata,
+                        ...parsedMetadata
+                    }
+                }
+                finaldocs.push(newdoc)
+            }
+            return finaldocs
+        }
+
+        return docs
+    }
+}
+
+interface AirtableLoaderParams {
+    baseId: string
+    tableId: string
+    accessToken: string
+    limit?: number
+    returnAll?: boolean
+}
+
+interface AirtableLoaderResponse {
+    records: AirtableLoaderPage[]
+    offset?: string
+}
+
+interface AirtableLoaderPage {
+    id: string
+    createdTime: string
+    fields: ICommonObject
+}
+
+class AirtableLoader extends BaseDocumentLoader {
+    public readonly baseId: string
+
+    public readonly tableId: string
+
+    public readonly accessToken: string
+
+    public readonly limit: number
+
+    public readonly returnAll: boolean
+
+    constructor({ baseId, tableId, accessToken, limit = 100, returnAll = false }: AirtableLoaderParams) {
+        super()
+        this.baseId = baseId
+        this.tableId = tableId
+        this.accessToken = accessToken
+        this.limit = limit
+        this.returnAll = returnAll
+    }
+
+    public async load(): Promise<Document[]> {
+        if (this.returnAll) {
+            return this.loadAll()
+        }
+        return this.loadLimit()
+    }
+
+    protected async fetchAirtableData(url: string, params: ICommonObject): Promise<AirtableLoaderResponse> {
+        try {
+            const headers = {
+                Authorization: `Bearer ${this.accessToken}`,
+                'Content-Type': 'application/json',
+                Accept: 'application/json'
+            }
+            const response = await axios.get(url, { params, headers })
+            return response.data
+        } catch (error) {
+            throw new Error(`Failed to fetch ${url} from Airtable: ${error}`)
+        }
+    }
+
+    private createDocumentFromPage(page: AirtableLoaderPage): Document {
+        // Generate the URL
+        const pageUrl = `https://api.airtable.com/v0/${this.baseId}/${this.tableId}/${page.id}`
+
+        // Return a langchain document
+        return new Document({
+            pageContent: JSON.stringify(page.fields, null, 2),
+            metadata: {
+                url: pageUrl
+            }
+        })
+    }
+
+    private async loadLimit(): Promise<Document[]> {
+        const params = { maxRecords: this.limit }
+        const data = await this.fetchAirtableData(`https://api.airtable.com/v0/${this.baseId}/${this.tableId}`, params)
+        if (data.records.length === 0) {
+            return []
+        }
+        return data.records.map((page) => this.createDocumentFromPage(page))
+    }
+
+    private async loadAll(): Promise<Document[]> {
+        const params: ICommonObject = { pageSize: 100 }
+        let data: AirtableLoaderResponse
+        let returnPages: AirtableLoaderPage[] = []
+
+        do {
+            data = await this.fetchAirtableData(`https://api.airtable.com/v0/${this.baseId}/${this.tableId}`, params)
+            returnPages.push.apply(returnPages, data.records)
+            params.offset = data.offset
+        } while (data.offset !== undefined)
+        return returnPages.map((page) => this.createDocumentFromPage(page))
+    }
+}
+
+module.exports = {
+    nodeClass: Airtable_DocumentLoaders
+}
@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<svg width="256px" height="215px" viewBox="0 0 256 215" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" preserveAspectRatio="xMidYMid">
+	<g>
+		<path d="M114.25873,2.70101695 L18.8604023,42.1756384 C13.5552723,44.3711638 13.6102328,51.9065311 18.9486282,54.0225085 L114.746142,92.0117514 C123.163769,95.3498757 132.537419,95.3498757 140.9536,92.0117514 L236.75256,54.0225085 C242.08951,51.9065311 242.145916,44.3711638 236.83934,42.1756384 L141.442459,2.70101695 C132.738459,-0.900338983 122.961284,-0.900338983 114.25873,2.70101695" fill="#FFBF00"></path>
+		<path d="M136.349071,112.756863 L136.349071,207.659101 C136.349071,212.173089 140.900664,215.263892 145.096461,213.600615 L251.844122,172.166219 C254.281184,171.200072 255.879376,168.845451 255.879376,166.224705 L255.879376,71.3224678 C255.879376,66.8084791 251.327783,63.7176768 247.131986,65.3809537 L140.384325,106.815349 C137.94871,107.781496 136.349071,110.136118 136.349071,112.756863" fill="#26B5F8"></path>
+		<path d="M111.422771,117.65355 L79.742409,132.949912 L76.5257763,134.504714 L9.65047684,166.548104 C5.4112904,168.593211 0.000578531073,165.503855 0.000578531073,160.794612 L0.000578531073,71.7210757 C0.000578531073,70.0173017 0.874160452,68.5463864 2.04568588,67.4384994 C2.53454463,66.9481944 3.08848814,66.5446689 3.66412655,66.2250305 C5.26231864,65.2661153 7.54173107,65.0101153 9.47981017,65.7766689 L110.890522,105.957098 C116.045234,108.002206 116.450206,115.225166 111.422771,117.65355" fill="#ED3049"></path>
+		<path d="M111.422771,117.65355 L79.742409,132.949912 L2.04568588,67.4384994 C2.53454463,66.9481944 3.08848814,66.5446689 3.66412655,66.2250305 C5.26231864,65.2661153 7.54173107,65.0101153 9.47981017,65.7766689 L110.890522,105.957098 C116.045234,108.002206 116.450206,115.225166 111.422771,117.65355" fill-opacity="0.25" fill="#000000"></path>
+	</g>
+</svg>
@@ -39,8 +39,7 @@ class NotionDB_DocumentLoaders implements INode {
                label: 'Notion Database Id',
                name: 'databaseId',
                type: 'string',
-                description:
-                    'If your URL looks like - https://www.notion.so/<long_hash_1>?v=<long_hash_2>, then <long_hash_1> is the database ID'
+                description: 'If your URL looks like - https://www.notion.so/abcdefh?v=long_hash_2, then abcdefh is the database ID'
            },
            {
                label: 'Metadata',