Feature/externalize files from chatflow - do not save as base64 (#1976)

* initial commit. Externalizing the file base64 string from flowData * csv - docloader - Externalizing the file base64 string from flowData * csv - docloader - Externalizing the file base64 string from flowData * DocX - docloader - Externalizing the file base64 string from flowData * Json - docloader - Externalizing the file base64 string from flowData * Jsonlines - docloader - Externalizing the file base64 string from flowData * PDF - docloader - Externalizing the file base64 string from flowData * Vectara - vector store - Externalizing the file base64 string from flowData * OpenAPIToolkit - tools - Externalizing the file base64 string from flowData * OpenAPIChain - chain - Externalizing the file base64 string from flowData * lint fixes * datasource enabled - initial commit * CSVAgent - agents - Externalizing the file base64 string from flowData * Externalizing the file base64 string from flowData * Externalizing the file base64 string from flowData * add pnpm-lock.yaml * update filerepository to add try catch * Rename FileRepository.ts to fileRepository.ts --------- Co-authored-by: Henry <hzj94@hotmail.com> Co-authored-by: Henry Heng <henryheng@flowiseai.com>
2026-06-28 13:00:56 +03:00 · 2024-04-04 21:41:06 +05:30
parent eed7de6df5
commit 658fa3984e
16 changed files with 593 additions and 194 deletions
@@ -1,6 +1,9 @@
-import { INode, INodeData, INodeParams } from '../../../src/Interface'
+import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
 import { TextSplitter } from 'langchain/text_splitter'
 import { CSVLoader } from 'langchain/document_loaders/fs/csv'
+import path from 'path'
+import { getStoragePath } from '../../../src'
+import fs from 'fs'

 class Csv_DocumentLoaders implements INode {
    label: string
@@ -53,7 +56,7 @@ class Csv_DocumentLoaders implements INode {
        ]
    }

-    async init(nodeData: INodeData): Promise<any> {
+    async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
        const textSplitter = nodeData.inputs?.textSplitter as TextSplitter
        const csvFileBase64 = nodeData.inputs?.csvFile as string
        const columnName = nodeData.inputs?.columnName as string
@@ -62,25 +65,50 @@ class Csv_DocumentLoaders implements INode {
        let alldocs = []
        let files: string[] = []

-        if (csvFileBase64.startsWith('[') && csvFileBase64.endsWith(']')) {
-            files = JSON.parse(csvFileBase64)
-        } else {
-            files = [csvFileBase64]
-        }
-
-        for (const file of files) {
-            const splitDataURI = file.split(',')
-            splitDataURI.pop()
-            const bf = Buffer.from(splitDataURI.pop() || '', 'base64')
-            const blob = new Blob([bf])
-            const loader = new CSVLoader(blob, columnName.trim().length === 0 ? undefined : columnName.trim())
-
-            if (textSplitter) {
-                const docs = await loader.loadAndSplit(textSplitter)
-                alldocs.push(...docs)
+        if (csvFileBase64.startsWith('FILE-STORAGE::')) {
+            const fileName = csvFileBase64.replace('FILE-STORAGE::', '')
+            if (fileName.startsWith('[') && fileName.endsWith(']')) {
+                files = JSON.parse(fileName)
            } else {
-                const docs = await loader.load()
-                alldocs.push(...docs)
+                files = [fileName]
+            }
+            const chatflowid = options.chatflowid
+
+            for (const file of files) {
+                const fileInStorage = path.join(getStoragePath(), chatflowid, file)
+                const fileData = fs.readFileSync(fileInStorage)
+                const blob = new Blob([fileData])
+                const loader = new CSVLoader(blob, columnName.trim().length === 0 ? undefined : columnName.trim())
+
+                if (textSplitter) {
+                    const docs = await loader.loadAndSplit(textSplitter)
+                    alldocs.push(...docs)
+                } else {
+                    const docs = await loader.load()
+                    alldocs.push(...docs)
+                }
+            }
+        } else {
+            if (csvFileBase64.startsWith('[') && csvFileBase64.endsWith(']')) {
+                files = JSON.parse(csvFileBase64)
+            } else {
+                files = [csvFileBase64]
+            }
+
+            for (const file of files) {
+                const splitDataURI = file.split(',')
+                splitDataURI.pop()
+                const bf = Buffer.from(splitDataURI.pop() || '', 'base64')
+                const blob = new Blob([bf])
+                const loader = new CSVLoader(blob, columnName.trim().length === 0 ? undefined : columnName.trim())
+
+                if (textSplitter) {
+                    const docs = await loader.loadAndSplit(textSplitter)
+                    alldocs.push(...docs)
+                } else {
+                    const docs = await loader.load()
+                    alldocs.push(...docs)
+                }
            }
        }

@@ -1,6 +1,9 @@
-import { INode, INodeData, INodeParams } from '../../../src/Interface'
+import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
 import { TextSplitter } from 'langchain/text_splitter'
 import { DocxLoader } from 'langchain/document_loaders/fs/docx'
+import path from 'path'
+import { getStoragePath } from '../../../src'
+import fs from 'fs'

 class Docx_DocumentLoaders implements INode {
    label: string
@@ -45,7 +48,7 @@ class Docx_DocumentLoaders implements INode {
        ]
    }

-    async init(nodeData: INodeData): Promise<any> {
+    async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
        const textSplitter = nodeData.inputs?.textSplitter as TextSplitter
        const docxFileBase64 = nodeData.inputs?.docxFile as string
        const metadata = nodeData.inputs?.metadata
@@ -53,25 +56,50 @@ class Docx_DocumentLoaders implements INode {
        let alldocs = []
        let files: string[] = []

-        if (docxFileBase64.startsWith('[') && docxFileBase64.endsWith(']')) {
-            files = JSON.parse(docxFileBase64)
-        } else {
-            files = [docxFileBase64]
-        }
-
-        for (const file of files) {
-            const splitDataURI = file.split(',')
-            splitDataURI.pop()
-            const bf = Buffer.from(splitDataURI.pop() || '', 'base64')
-            const blob = new Blob([bf])
-            const loader = new DocxLoader(blob)
-
-            if (textSplitter) {
-                const docs = await loader.loadAndSplit(textSplitter)
-                alldocs.push(...docs)
+        if (docxFileBase64.startsWith('FILE-STORAGE::')) {
+            const fileName = docxFileBase64.replace('FILE-STORAGE::', '')
+            if (fileName.startsWith('[') && fileName.endsWith(']')) {
+                files = JSON.parse(fileName)
            } else {
-                const docs = await loader.load()
-                alldocs.push(...docs)
+                files = [fileName]
+            }
+            const chatflowid = options.chatflowid
+
+            for (const file of files) {
+                const fileInStorage = path.join(getStoragePath(), chatflowid, file)
+                const fileData = fs.readFileSync(fileInStorage)
+                const blob = new Blob([fileData])
+                const loader = new DocxLoader(blob)
+
+                if (textSplitter) {
+                    const docs = await loader.loadAndSplit(textSplitter)
+                    alldocs.push(...docs)
+                } else {
+                    const docs = await loader.load()
+                    alldocs.push(...docs)
+                }
+            }
+        } else {
+            if (docxFileBase64.startsWith('[') && docxFileBase64.endsWith(']')) {
+                files = JSON.parse(docxFileBase64)
+            } else {
+                files = [docxFileBase64]
+            }
+
+            for (const file of files) {
+                const splitDataURI = file.split(',')
+                splitDataURI.pop()
+                const bf = Buffer.from(splitDataURI.pop() || '', 'base64')
+                const blob = new Blob([bf])
+                const loader = new DocxLoader(blob)
+
+                if (textSplitter) {
+                    const docs = await loader.loadAndSplit(textSplitter)
+                    alldocs.push(...docs)
+                } else {
+                    const docs = await loader.load()
+                    alldocs.push(...docs)
+                }
            }
        }

@@ -1,6 +1,9 @@
-import { INode, INodeData, INodeParams } from '../../../src/Interface'
+import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
 import { TextSplitter } from 'langchain/text_splitter'
 import { JSONLoader } from 'langchain/document_loaders/fs/json'
+import { getStoragePath } from '../../../src'
+import fs from 'fs'
+import path from 'path'

 class Json_DocumentLoaders implements INode {
    label: string
@@ -53,7 +56,7 @@ class Json_DocumentLoaders implements INode {
        ]
    }

-    async init(nodeData: INodeData): Promise<any> {
+    async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
        const textSplitter = nodeData.inputs?.textSplitter as TextSplitter
        const jsonFileBase64 = nodeData.inputs?.jsonFile as string
        const pointersName = nodeData.inputs?.pointersName as string
@@ -68,25 +71,51 @@ class Json_DocumentLoaders implements INode {
        let alldocs = []
        let files: string[] = []

-        if (jsonFileBase64.startsWith('[') && jsonFileBase64.endsWith(']')) {
-            files = JSON.parse(jsonFileBase64)
-        } else {
-            files = [jsonFileBase64]
-        }
-
-        for (const file of files) {
-            const splitDataURI = file.split(',')
-            splitDataURI.pop()
-            const bf = Buffer.from(splitDataURI.pop() || '', 'base64')
-            const blob = new Blob([bf])
-            const loader = new JSONLoader(blob, pointers.length != 0 ? pointers : undefined)
-
-            if (textSplitter) {
-                const docs = await loader.loadAndSplit(textSplitter)
-                alldocs.push(...docs)
+        //FILE-STORAGE::["CONTRIBUTING.md","LICENSE.md","README.md"]
+        if (jsonFileBase64.startsWith('FILE-STORAGE::')) {
+            const fileName = jsonFileBase64.replace('FILE-STORAGE::', '')
+            if (fileName.startsWith('[') && fileName.endsWith(']')) {
+                files = JSON.parse(fileName)
            } else {
-                const docs = await loader.load()
-                alldocs.push(...docs)
+                files = [fileName]
+            }
+            const chatflowid = options.chatflowid
+
+            for (const file of files) {
+                const fileInStorage = path.join(getStoragePath(), chatflowid, file)
+                const fileData = fs.readFileSync(fileInStorage)
+                const blob = new Blob([fileData])
+                const loader = new JSONLoader(blob, pointers.length != 0 ? pointers : undefined)
+
+                if (textSplitter) {
+                    const docs = await loader.loadAndSplit(textSplitter)
+                    alldocs.push(...docs)
+                } else {
+                    const docs = await loader.load()
+                    alldocs.push(...docs)
+                }
+            }
+        } else {
+            if (jsonFileBase64.startsWith('[') && jsonFileBase64.endsWith(']')) {
+                files = JSON.parse(jsonFileBase64)
+            } else {
+                files = [jsonFileBase64]
+            }
+
+            for (const file of files) {
+                const splitDataURI = file.split(',')
+                splitDataURI.pop()
+                const bf = Buffer.from(splitDataURI.pop() || '', 'base64')
+                const blob = new Blob([bf])
+                const loader = new JSONLoader(blob, pointers.length != 0 ? pointers : undefined)
+
+                if (textSplitter) {
+                    const docs = await loader.loadAndSplit(textSplitter)
+                    alldocs.push(...docs)
+                } else {
+                    const docs = await loader.load()
+                    alldocs.push(...docs)
+                }
            }
        }

@@ -1,6 +1,9 @@
-import { INode, INodeData, INodeParams } from '../../../src/Interface'
+import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
 import { TextSplitter } from 'langchain/text_splitter'
 import { JSONLinesLoader } from 'langchain/document_loaders/fs/json'
+import { getStoragePath } from '../../../src'
+import fs from 'fs'
+import path from 'path'

 class Jsonlines_DocumentLoaders implements INode {
    label: string
@@ -52,7 +55,7 @@ class Jsonlines_DocumentLoaders implements INode {
        ]
    }

-    async init(nodeData: INodeData): Promise<any> {
+    async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
        const textSplitter = nodeData.inputs?.textSplitter as TextSplitter
        const jsonLinesFileBase64 = nodeData.inputs?.jsonlinesFile as string
        const pointerName = nodeData.inputs?.pointerName as string
@@ -62,26 +65,51 @@ class Jsonlines_DocumentLoaders implements INode {
        let files: string[] = []

        let pointer = '/' + pointerName.trim()
-
-        if (jsonLinesFileBase64.startsWith('[') && jsonLinesFileBase64.endsWith(']')) {
-            files = JSON.parse(jsonLinesFileBase64)
-        } else {
-            files = [jsonLinesFileBase64]
-        }
-
-        for (const file of files) {
-            const splitDataURI = file.split(',')
-            splitDataURI.pop()
-            const bf = Buffer.from(splitDataURI.pop() || '', 'base64')
-            const blob = new Blob([bf])
-            const loader = new JSONLinesLoader(blob, pointer)
-
-            if (textSplitter) {
-                const docs = await loader.loadAndSplit(textSplitter)
-                alldocs.push(...docs)
+        //FILE-STORAGE::["CONTRIBUTING.md","LICENSE.md","README.md"]
+        if (jsonLinesFileBase64.startsWith('FILE-STORAGE::')) {
+            const fileName = jsonLinesFileBase64.replace('FILE-STORAGE::', '')
+            if (fileName.startsWith('[') && fileName.endsWith(']')) {
+                files = JSON.parse(fileName)
            } else {
-                const docs = await loader.load()
-                alldocs.push(...docs)
+                files = [fileName]
+            }
+            const chatflowid = options.chatflowid
+
+            for (const file of files) {
+                const fileInStorage = path.join(getStoragePath(), chatflowid, file)
+                const fileData = fs.readFileSync(fileInStorage)
+                const blob = new Blob([fileData])
+                const loader = new JSONLinesLoader(blob, pointer)
+
+                if (textSplitter) {
+                    const docs = await loader.loadAndSplit(textSplitter)
+                    alldocs.push(...docs)
+                } else {
+                    const docs = await loader.load()
+                    alldocs.push(...docs)
+                }
+            }
+        } else {
+            if (jsonLinesFileBase64.startsWith('[') && jsonLinesFileBase64.endsWith(']')) {
+                files = JSON.parse(jsonLinesFileBase64)
+            } else {
+                files = [jsonLinesFileBase64]
+            }
+
+            for (const file of files) {
+                const splitDataURI = file.split(',')
+                splitDataURI.pop()
+                const bf = Buffer.from(splitDataURI.pop() || '', 'base64')
+                const blob = new Blob([bf])
+                const loader = new JSONLinesLoader(blob, pointer)
+
+                if (textSplitter) {
+                    const docs = await loader.loadAndSplit(textSplitter)
+                    alldocs.push(...docs)
+                } else {
+                    const docs = await loader.load()
+                    alldocs.push(...docs)
+                }
            }
        }

@@ -1,6 +1,9 @@
-import { INode, INodeData, INodeParams } from '../../../src/Interface'
+import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
 import { TextSplitter } from 'langchain/text_splitter'
 import { PDFLoader } from 'langchain/document_loaders/fs/pdf'
+import { getStoragePath } from '../../../src'
+import fs from 'fs'
+import path from 'path'

 class Pdf_DocumentLoaders implements INode {
    label: string
@@ -68,53 +71,44 @@ class Pdf_DocumentLoaders implements INode {
        ]
    }

-    async init(nodeData: INodeData): Promise<any> {
+    async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
        const textSplitter = nodeData.inputs?.textSplitter as TextSplitter
        const pdfFileBase64 = nodeData.inputs?.pdfFile as string
        const usage = nodeData.inputs?.usage as string
        const metadata = nodeData.inputs?.metadata
        const legacyBuild = nodeData.inputs?.legacyBuild as boolean

-        let alldocs = []
+        let alldocs: any[] = []
        let files: string[] = []

-        if (pdfFileBase64.startsWith('[') && pdfFileBase64.endsWith(']')) {
-            files = JSON.parse(pdfFileBase64)
-        } else {
-            files = [pdfFileBase64]
-        }
-
-        for (const file of files) {
-            const splitDataURI = file.split(',')
-            splitDataURI.pop()
-            const bf = Buffer.from(splitDataURI.pop() || '', 'base64')
-            if (usage === 'perFile') {
-                const loader = new PDFLoader(new Blob([bf]), {
-                    splitPages: false,
-                    pdfjs: () =>
-                        // @ts-ignore
-                        legacyBuild ? import('pdfjs-dist/legacy/build/pdf.js') : import('pdf-parse/lib/pdf.js/v1.10.100/build/pdf.js')
-                })
-                if (textSplitter) {
-                    const docs = await loader.loadAndSplit(textSplitter)
-                    alldocs.push(...docs)
-                } else {
-                    const docs = await loader.load()
-                    alldocs.push(...docs)
-                }
+        //FILE-STORAGE::["CONTRIBUTING.md","LICENSE.md","README.md"]
+        if (pdfFileBase64.startsWith('FILE-STORAGE::')) {
+            const fileName = pdfFileBase64.replace('FILE-STORAGE::', '')
+            if (fileName.startsWith('[') && fileName.endsWith(']')) {
+                files = JSON.parse(fileName)
            } else {
-                const loader = new PDFLoader(new Blob([bf]), {
-                    pdfjs: () =>
-                        // @ts-ignore
-                        legacyBuild ? import('pdfjs-dist/legacy/build/pdf.js') : import('pdf-parse/lib/pdf.js/v1.10.100/build/pdf.js')
-                })
-                if (textSplitter) {
-                    const docs = await loader.loadAndSplit(textSplitter)
-                    alldocs.push(...docs)
-                } else {
-                    const docs = await loader.load()
-                    alldocs.push(...docs)
-                }
+                files = [fileName]
+            }
+            const chatflowid = options.chatflowid
+
+            for (const file of files) {
+                const fileInStorage = path.join(getStoragePath(), chatflowid, file)
+                const fileData = fs.readFileSync(fileInStorage)
+                const bf = Buffer.from(fileData)
+                await this.extractDocs(usage, bf, legacyBuild, textSplitter, alldocs)
+            }
+        } else {
+            if (pdfFileBase64.startsWith('[') && pdfFileBase64.endsWith(']')) {
+                files = JSON.parse(pdfFileBase64)
+            } else {
+                files = [pdfFileBase64]
+            }
+
+            for (const file of files) {
+                const splitDataURI = file.split(',')
+                splitDataURI.pop()
+                const bf = Buffer.from(splitDataURI.pop() || '', 'base64')
+                await this.extractDocs(usage, bf, legacyBuild, textSplitter, alldocs)
            }
        }

@@ -136,6 +130,37 @@ class Pdf_DocumentLoaders implements INode {

        return alldocs
    }
+
+    private async extractDocs(usage: string, bf: Buffer, legacyBuild: boolean, textSplitter: TextSplitter, alldocs: any[]) {
+        if (usage === 'perFile') {
+            const loader = new PDFLoader(new Blob([bf]), {
+                splitPages: false,
+                pdfjs: () =>
+                    // @ts-ignore
+                    legacyBuild ? import('pdfjs-dist/legacy/build/pdf.js') : import('pdf-parse/lib/pdf.js/v1.10.100/build/pdf.js')
+            })
+            if (textSplitter) {
+                const docs = await loader.loadAndSplit(textSplitter)
+                alldocs.push(...docs)
+            } else {
+                const docs = await loader.load()
+                alldocs.push(...docs)
+            }
+        } else {
+            const loader = new PDFLoader(new Blob([bf]), {
+                pdfjs: () =>
+                    // @ts-ignore
+                    legacyBuild ? import('pdfjs-dist/legacy/build/pdf.js') : import('pdf-parse/lib/pdf.js/v1.10.100/build/pdf.js')
+            })
+            if (textSplitter) {
+                const docs = await loader.loadAndSplit(textSplitter)
+                alldocs.push(...docs)
+            } else {
+                const docs = await loader.load()
+                alldocs.push(...docs)
+            }
+        }
+    }
 }

 module.exports = { nodeClass: Pdf_DocumentLoaders }
@@ -1,8 +1,10 @@
-import { INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
+import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
 import { TextSplitter } from 'langchain/text_splitter'
 import { TextLoader } from 'langchain/document_loaders/fs/text'
 import { Document } from '@langchain/core/documents'
-import { handleEscapeCharacters } from '../../../src'
+import { getStoragePath, handleEscapeCharacters } from '../../../src'
+import fs from 'fs'
+import path from 'path'

 class Text_DocumentLoaders implements INode {
    label: string
@@ -63,7 +65,7 @@ class Text_DocumentLoaders implements INode {
        ]
    }

-    async init(nodeData: INodeData): Promise<any> {
+    async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
        const textSplitter = nodeData.inputs?.textSplitter as TextSplitter
        const txtFileBase64 = nodeData.inputs?.txtFile as string
        const metadata = nodeData.inputs?.metadata
@@ -72,25 +74,51 @@ class Text_DocumentLoaders implements INode {
        let alldocs = []
        let files: string[] = []

-        if (txtFileBase64.startsWith('[') && txtFileBase64.endsWith(']')) {
-            files = JSON.parse(txtFileBase64)
-        } else {
-            files = [txtFileBase64]
-        }
-
-        for (const file of files) {
-            const splitDataURI = file.split(',')
-            splitDataURI.pop()
-            const bf = Buffer.from(splitDataURI.pop() || '', 'base64')
-            const blob = new Blob([bf])
-            const loader = new TextLoader(blob)
-
-            if (textSplitter) {
-                const docs = await loader.loadAndSplit(textSplitter)
-                alldocs.push(...docs)
+        //FILE-STORAGE::["CONTRIBUTING.md","LICENSE.md","README.md"]
+        if (txtFileBase64.startsWith('FILE-STORAGE::')) {
+            const fileName = txtFileBase64.replace('FILE-STORAGE::', '')
+            if (fileName.startsWith('[') && fileName.endsWith(']')) {
+                files = JSON.parse(fileName)
            } else {
-                const docs = await loader.load()
-                alldocs.push(...docs)
+                files = [fileName]
+            }
+            const chatflowid = options.chatflowid
+
+            for (const file of files) {
+                const fileInStorage = path.join(getStoragePath(), chatflowid, file)
+                const fileData = fs.readFileSync(fileInStorage)
+                const blob = new Blob([fileData])
+                const loader = new TextLoader(blob)
+
+                if (textSplitter) {
+                    const docs = await loader.loadAndSplit(textSplitter)
+                    alldocs.push(...docs)
+                } else {
+                    const docs = await loader.load()
+                    alldocs.push(...docs)
+                }
+            }
+        } else {
+            if (txtFileBase64.startsWith('[') && txtFileBase64.endsWith(']')) {
+                files = JSON.parse(txtFileBase64)
+            } else {
+                files = [txtFileBase64]
+            }
+
+            for (const file of files) {
+                const splitDataURI = file.split(',')
+                splitDataURI.pop()
+                const bf = Buffer.from(splitDataURI.pop() || '', 'base64')
+                const blob = new Blob([bf])
+                const loader = new TextLoader(blob)
+
+                if (textSplitter) {
+                    const docs = await loader.loadAndSplit(textSplitter)
+                    alldocs.push(...docs)
+                } else {
+                    const docs = await loader.load()
+                    alldocs.push(...docs)
+                }
            }
        }