farcasclaudiu/Flowise
1
0
Fork 0
mirror of https://github.com/farcasclaudiu/Flowise.git synced 2026-06-28 19:00:59 +03:00
Code Issues Packages Projects Releases Wiki Activity

Feature/s3 storage (#2226)

Browse Source 
* centralizing file writing....

* allowing s3 as storage option

* allowing s3 as storage option

* update s3 storage

---------

Co-authored-by: Henry <hzj94@hotmail.com>
This commit is contained in:
Vinod Kiran
2024-04-23 16:05:38 +05:30
committed by GitHub
parent 6ab259b6aa
commit 7006d64de0
34 changed files with 458 additions and 257 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/components/nodes/documentloaders/Csv/Csv.ts
+2 -5
View File
@@ -1,9 +1,7 @@
import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
import { TextSplitter } from 'langchain/text_splitter'
import { CSVLoader } from 'langchain/document_loaders/fs/csv'
import path from 'path'
import { getStoragePath } from '../../../src'
import fs from 'fs'
import { getFileFromStorage } from '../../../src'
class Csv_DocumentLoaders implements INode {
label: string
@@ -75,8 +73,7 @@ class Csv_DocumentLoaders implements INode {
const chatflowid = options.chatflowid
for (const file of files) {
const fileInStorage = path.join(getStoragePath(), chatflowid, file)
const fileData = fs.readFileSync(fileInStorage)
const fileData = await getFileFromStorage(file, chatflowid)
const blob = new Blob([fileData])
const loader = new CSVLoader(blob, columnName.trim().length === 0 ? undefined : columnName.trim())
packages/components/nodes/documentloaders/Docx/Docx.ts
+2 -5
View File
@@ -1,9 +1,7 @@
import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
import { TextSplitter } from 'langchain/text_splitter'
import { DocxLoader } from 'langchain/document_loaders/fs/docx'
import path from 'path'
import { getStoragePath } from '../../../src'
import fs from 'fs'
import { getFileFromStorage } from '../../../src'
class Docx_DocumentLoaders implements INode {
label: string
@@ -66,8 +64,7 @@ class Docx_DocumentLoaders implements INode {
const chatflowid = options.chatflowid
for (const file of files) {
const fileInStorage = path.join(getStoragePath(), chatflowid, file)
const fileData = fs.readFileSync(fileInStorage)
const fileData = await getFileFromStorage(file, chatflowid)
const blob = new Blob([fileData])
const loader = new DocxLoader(blob)
packages/components/nodes/documentloaders/Json/Json.ts
+2 -5
View File
@@ -1,9 +1,7 @@
import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
import { TextSplitter } from 'langchain/text_splitter'
import { JSONLoader } from 'langchain/document_loaders/fs/json'
import { getStoragePath } from '../../../src'
import fs from 'fs'
import path from 'path'
import { getFileFromStorage } from '../../../src'
class Json_DocumentLoaders implements INode {
label: string
@@ -82,8 +80,7 @@ class Json_DocumentLoaders implements INode {
const chatflowid = options.chatflowid
for (const file of files) {
const fileInStorage = path.join(getStoragePath(), chatflowid, file)
const fileData = fs.readFileSync(fileInStorage)
const fileData = await getFileFromStorage(file, chatflowid)
const blob = new Blob([fileData])
const loader = new JSONLoader(blob, pointers.length != 0 ? pointers : undefined)
packages/components/nodes/documentloaders/Jsonlines/Jsonlines.ts
+2 -5
View File
@@ -1,9 +1,7 @@
import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
import { TextSplitter } from 'langchain/text_splitter'
import { JSONLinesLoader } from 'langchain/document_loaders/fs/json'
import { getStoragePath } from '../../../src'
import fs from 'fs'
import path from 'path'
import { getFileFromStorage } from '../../../src'
class Jsonlines_DocumentLoaders implements INode {
label: string
@@ -76,8 +74,7 @@ class Jsonlines_DocumentLoaders implements INode {
const chatflowid = options.chatflowid
for (const file of files) {
const fileInStorage = path.join(getStoragePath(), chatflowid, file)
const fileData = fs.readFileSync(fileInStorage)
const fileData = await getFileFromStorage(file, chatflowid)
const blob = new Blob([fileData])
const loader = new JSONLinesLoader(blob, pointer)
packages/components/nodes/documentloaders/Pdf/Pdf.ts
+2 -5
View File
@@ -1,9 +1,7 @@
import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
import { TextSplitter } from 'langchain/text_splitter'
import { PDFLoader } from 'langchain/document_loaders/fs/pdf'
import { getStoragePath } from '../../../src'
import fs from 'fs'
import path from 'path'
import { getFileFromStorage } from '../../../src'
class Pdf_DocumentLoaders implements INode {
label: string
@@ -92,8 +90,7 @@ class Pdf_DocumentLoaders implements INode {
const chatflowid = options.chatflowid
for (const file of files) {
const fileInStorage = path.join(getStoragePath(), chatflowid, file)
const fileData = fs.readFileSync(fileInStorage)
const fileData = await getFileFromStorage(file, chatflowid)
const bf = Buffer.from(fileData)
await this.extractDocs(usage, bf, legacyBuild, textSplitter, alldocs)
}
packages/components/nodes/documentloaders/Text/Text.ts
+2 -5
View File
@@ -2,9 +2,7 @@ import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from
import { TextSplitter } from 'langchain/text_splitter'
import { TextLoader } from 'langchain/document_loaders/fs/text'
import { Document } from '@langchain/core/documents'
import { getStoragePath, handleEscapeCharacters } from '../../../src'
import fs from 'fs'
import path from 'path'
import { getFileFromStorage, handleEscapeCharacters } from '../../../src'
class Text_DocumentLoaders implements INode {
label: string
@@ -85,8 +83,7 @@ class Text_DocumentLoaders implements INode {
const chatflowid = options.chatflowid
for (const file of files) {
const fileInStorage = path.join(getStoragePath(), chatflowid, file)
const fileData = fs.readFileSync(fileInStorage)
const fileData = await getFileFromStorage(file, chatflowid)
const blob = new Blob([fileData])
const loader = new TextLoader(blob)