add api config

packages/components/nodes/documentloaders/Csv/Csv.ts

@@ -1,6 +1,7 @@
 import { INode, INodeData, INodeParams } from '../../../src/Interface'
 import { TextSplitter } from 'langchain/text_splitter'
 import { CSVLoader } from 'langchain/document_loaders/fs/csv'
+import { getBlob } from '../../../src/utils'
 
 class Csv_DocumentLoaders implements INode {
     label: string
@@ -48,11 +49,8 @@ class Csv_DocumentLoaders implements INode {
         const textSplitter = nodeData.inputs?.textSplitter as TextSplitter
         const csvFileBase64 = nodeData.inputs?.csvFile as string
         const columnName = nodeData.inputs?.columnName as string
-        const splitDataURI = csvFileBase64.split(',')
-        splitDataURI.pop()
-        const bf = Buffer.from(splitDataURI.pop() || '', 'base64')
 
-        const blob = new Blob([bf])
+        const blob = new Blob(getBlob(csvFileBase64))
         const loader = new CSVLoader(blob, columnName.trim().length === 0 ? undefined : columnName.trim())
 
         if (textSplitter) {

packages/components/nodes/documentloaders/Docx/Docx.ts

@@ -1,6 +1,7 @@
 import { INode, INodeData, INodeParams } from '../../../src/Interface'
 import { TextSplitter } from 'langchain/text_splitter'
 import { DocxLoader } from 'langchain/document_loaders/fs/docx'
+import { getBlob } from '../../../src/utils'
 
 class Docx_DocumentLoaders implements INode {
     label: string
@@ -39,11 +40,8 @@ class Docx_DocumentLoaders implements INode {
     async init(nodeData: INodeData): Promise<any> {
         const textSplitter = nodeData.inputs?.textSplitter as TextSplitter
         const docxFileBase64 = nodeData.inputs?.docxFile as string
-        const splitDataURI = docxFileBase64.split(',')
-        splitDataURI.pop()
-        const bf = Buffer.from(splitDataURI.pop() || '', 'base64')
 
-        const blob = new Blob([bf])
+        const blob = new Blob(getBlob(docxFileBase64))
         const loader = new DocxLoader(blob)
 
         if (textSplitter) {

packages/components/nodes/documentloaders/Json/Json.ts

@@ -1,6 +1,7 @@
 import { INode, INodeData, INodeParams } from '../../../src/Interface'
 import { TextSplitter } from 'langchain/text_splitter'
 import { JSONLoader } from 'langchain/document_loaders/fs/json'
+import { getBlob } from '../../../src/utils'
 
 class Json_DocumentLoaders implements INode {
     label: string
@@ -48,9 +49,6 @@ class Json_DocumentLoaders implements INode {
         const textSplitter = nodeData.inputs?.textSplitter as TextSplitter
         const jsonFileBase64 = nodeData.inputs?.jsonFile as string
         const pointersName = nodeData.inputs?.pointersName as string
-        const splitDataURI = jsonFileBase64.split(',')
-        splitDataURI.pop()
-        const bf = Buffer.from(splitDataURI.pop() || '', 'base64')
 
         let pointers: string[] = []
         if (pointersName) {
@@ -58,7 +56,7 @@ class Json_DocumentLoaders implements INode {
             pointers = outputString.split(',').map((pointer) => '/' + pointer.trim())
         }
 
-        const blob = new Blob([bf])
+        const blob = new Blob(getBlob(jsonFileBase64))
         const loader = new JSONLoader(blob, pointers.length != 0 ? pointers : undefined)
 
         if (textSplitter) {

packages/components/nodes/documentloaders/Pdf/Pdf.ts

@@ -1,6 +1,7 @@
 import { INode, INodeData, INodeParams } from '../../../src/Interface'
 import { TextSplitter } from 'langchain/text_splitter'
 import { PDFLoader } from 'langchain/document_loaders/fs/pdf'
+import { getBlob } from '../../../src/utils'
 
 class Pdf_DocumentLoaders implements INode {
     label: string
@@ -57,10 +58,7 @@ class Pdf_DocumentLoaders implements INode {
         const pdfFileBase64 = nodeData.inputs?.pdfFile as string
         const usage = nodeData.inputs?.usage as string
 
-        const splitDataURI = pdfFileBase64.split(',')
-        splitDataURI.pop()
-        const bf = Buffer.from(splitDataURI.pop() || '', 'base64')
-        const blob = new Blob([bf])
+        const blob = new Blob(getBlob(pdfFileBase64))
 
         if (usage === 'perFile') {
             // @ts-ignore

packages/components/nodes/documentloaders/Text/Text.ts

@@ -1,6 +1,7 @@
 import { INode, INodeData, INodeParams } from '../../../src/Interface'
 import { TextSplitter } from 'langchain/text_splitter'
 import { TextLoader } from 'langchain/document_loaders/fs/text'
+import { getBlob } from '../../../src/utils'
 
 class Text_DocumentLoaders implements INode {
     label: string
@@ -39,11 +40,8 @@ class Text_DocumentLoaders implements INode {
     async init(nodeData: INodeData): Promise<any> {
         const textSplitter = nodeData.inputs?.textSplitter as TextSplitter
         const txtFileBase64 = nodeData.inputs?.txtFile as string
-        const splitDataURI = txtFileBase64.split(',')
-        splitDataURI.pop()
-        const bf = Buffer.from(splitDataURI.pop() || '', 'base64')
 
-        const blob = new Blob([bf])
+        const blob = new Blob(getBlob(txtFileBase64))
         const loader = new TextLoader(blob)
 
         if (textSplitter) {

packages/components/src/utils.ts

@@ -149,3 +149,27 @@ export const getInputVariables = (paramValue: string): string[] => {
     }
     return inputVariables
 }
+
+/**
+ * Get blob
+ * @param {string} fileBase64Str
+ * @returns {Buffer[]}
+ */
+export const getBlob = (fileBase64Str: string) => {
+    let bufferArray: Buffer[] = []
+    let files: string[] = []
+
+    if (fileBase64Str.startsWith('[') && fileBase64Str.endsWith(']')) {
+        files = JSON.parse(fileBase64Str)
+    } else {
+        files = [fileBase64Str]
+    }
+
+    for (const file of files) {
+        const splitDataURI = file.split(',')
+        splitDataURI.pop()
+        const bf = Buffer.from(splitDataURI.pop() || '', 'base64')
+        bufferArray.push(bf)
+    }
+    return bufferArray
+}