farcasclaudiu/Flowise
1
0
Fork 0
mirror of https://github.com/farcasclaudiu/Flowise.git synced 2026-06-29 11:01:18 +03:00
Code Issues Packages Projects Releases Wiki Activity

Feature/s3 storage (#2226)

Browse Source 
* centralizing file writing....

* allowing s3 as storage option

* allowing s3 as storage option

* update s3 storage

---------

Co-authored-by: Henry <hzj94@hotmail.com>
This commit is contained in:
Vinod Kiran
2024-04-23 16:05:38 +05:30
committed by GitHub
parent 6ab259b6aa
commit 7006d64de0
34 changed files with 458 additions and 257 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/components/nodes/agents/CSVAgent/CSVAgent.ts
+2 -5
View File
@@ -7,9 +7,7 @@ import { getBaseClasses } from '../../../src/utils'
import { LoadPyodide, finalSystemPrompt, systemPrompt } from './core'
import { checkInputs, Moderation } from '../../moderation/Moderation'
import { formatResponse } from '../../outputparsers/OutputParserHelpers'
import path from 'path'
import { getStoragePath } from '../../../src'
import fs from 'fs'
import { getFileFromStorage } from '../../../src'
class CSV_Agents implements INode {
label: string
@@ -114,8 +112,7 @@ class CSV_Agents implements INode {
const chatflowid = options.chatflowid
for (const file of files) {
const fileInStorage = path.join(getStoragePath(), chatflowid, file)
const fileData = fs.readFileSync(fileInStorage)
const fileData = await getFileFromStorage(file, chatflowid)
base64String += fileData.toString('base64')
}
} else {
packages/components/nodes/agents/ConversationalAgent/ConversationalAgent.ts
+1 -1
View File
@@ -203,7 +203,7 @@ const prepareAgent = async (
if (llmSupportsVision(model)) {
const visionChatModel = model as IVisionChatModal
const messageContent = addImagesToMessages(nodeData, options, model.multiModalOption)
const messageContent = await addImagesToMessages(nodeData, options, model.multiModalOption)
if (messageContent?.length) {
visionChatModel.setVisionModel()
packages/components/nodes/agents/ReActAgentChat/ReActAgentChat.ts
+1 -1
View File
@@ -98,7 +98,7 @@ class ReActAgentChat_Agents implements INode {
if (llmSupportsVision(model)) {
const visionChatModel = model as IVisionChatModal
const messageContent = addImagesToMessages(nodeData, options, model.multiModalOption)
const messageContent = await addImagesToMessages(nodeData, options, model.multiModalOption)
if (messageContent?.length) {
// Change model to vision supported
packages/components/nodes/agents/ToolAgent/ToolAgent.ts
+7 -3
View File
@@ -106,7 +106,7 @@ class ToolAgent_Agents implements INode {
}
}
const executor = prepareAgent(nodeData, options, { sessionId: this.sessionId, chatId: options.chatId, input })
const executor = await prepareAgent(nodeData, options, { sessionId: this.sessionId, chatId: options.chatId, input })
const loggerHandler = new ConsoleCallbackHandler(options.logger)
const callbacks = await additionalCallbacks(nodeData, options)
@@ -178,7 +178,11 @@ class ToolAgent_Agents implements INode {
}
}
const prepareAgent = (nodeData: INodeData, options: ICommonObject, flowObj: { sessionId?: string; chatId?: string; input?: string }) => {
const prepareAgent = async (
nodeData: INodeData,
options: ICommonObject,
flowObj: { sessionId?: string; chatId?: string; input?: string }
) => {
const model = nodeData.inputs?.model as BaseChatModel
const maxIterations = nodeData.inputs?.maxIterations as string
const memory = nodeData.inputs?.memory as FlowiseMemory
@@ -197,7 +201,7 @@ const prepareAgent = (nodeData: INodeData, options: ICommonObject, flowObj: { se
if (llmSupportsVision(model)) {
const visionChatModel = model as IVisionChatModal
const messageContent = addImagesToMessages(nodeData, options, model.multiModalOption)
const messageContent = await addImagesToMessages(nodeData, options, model.multiModalOption)
if (messageContent?.length) {
visionChatModel.setVisionModel()
packages/components/nodes/chains/ApiChain/OpenAPIChain.ts
+2 -5
View File
@@ -5,9 +5,7 @@ import { getBaseClasses } from '../../../src/utils'
import { ConsoleCallbackHandler, CustomChainHandler, additionalCallbacks } from '../../../src/handler'
import { checkInputs, Moderation, streamResponse } from '../../moderation/Moderation'
import { formatResponse } from '../../outputparsers/OutputParserHelpers'
import { getStoragePath } from '../../../src'
import fs from 'fs'
import path from 'path'
import { getFileFromStorage } from '../../../src'
class OpenApiChain_Chains implements INode {
label: string
@@ -111,8 +109,7 @@ const initChain = async (nodeData: INodeData, options: ICommonObject) => {
if (yamlFileBase64.startsWith('FILE-STORAGE::')) {
const file = yamlFileBase64.replace('FILE-STORAGE::', '')
const chatflowid = options.chatflowid
const fileInStorage = path.join(getStoragePath(), chatflowid, file)
const fileData = fs.readFileSync(fileInStorage)
const fileData = await getFileFromStorage(file, chatflowid)
yamlString = fileData.toString()
} else {
const splitDataURI = yamlFileBase64.split(',')
packages/components/nodes/chains/ConversationChain/ConversationChain.ts
+3 -3
View File
@@ -111,7 +111,7 @@ class ConversationChain_Chains implements INode {
async run(nodeData: INodeData, input: string, options: ICommonObject): Promise<string | object> {
const memory = nodeData.inputs?.memory
const chain = prepareChain(nodeData, options, this.sessionId)
const chain = await prepareChain(nodeData, options, this.sessionId)
const moderations = nodeData.inputs?.inputModeration as Moderation[]
if (moderations && moderations.length > 0) {
@@ -216,14 +216,14 @@ const prepareChatPrompt = (nodeData: INodeData, humanImageMessages: MessageConte
return chatPrompt
}
const prepareChain = (nodeData: INodeData, options: ICommonObject, sessionId?: string) => {
const prepareChain = async (nodeData: INodeData, options: ICommonObject, sessionId?: string) => {
let model = nodeData.inputs?.model as BaseChatModel
const memory = nodeData.inputs?.memory as FlowiseMemory
const memoryKey = memory.memoryKey ?? 'chat_history'
let messageContent: MessageContentImageUrl[] = []
if (llmSupportsVision(model)) {
messageContent = addImagesToMessages(nodeData, options, model.multiModalOption)
messageContent = await addImagesToMessages(nodeData, options, model.multiModalOption)
const visionChatModel = model as IVisionChatModal
if (messageContent?.length) {
visionChatModel.setVisionModel()
packages/components/nodes/chains/LLMChain/LLMChain.ts
+1 -1
View File
@@ -184,7 +184,7 @@ const runPrediction = async (
if (llmSupportsVision(chain.llm)) {
const visionChatModel = chain.llm as IVisionChatModal
const messageContent = addImagesToMessages(nodeData, options, visionChatModel.multiModalOption)
const messageContent = await addImagesToMessages(nodeData, options, visionChatModel.multiModalOption)
if (messageContent?.length) {
// Change model to gpt-4-vision && max token to higher when using gpt-4-vision
visionChatModel.setVisionModel()
packages/components/nodes/documentloaders/Csv/Csv.ts
+2 -5
View File
@@ -1,9 +1,7 @@
import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
import { TextSplitter } from 'langchain/text_splitter'
import { CSVLoader } from 'langchain/document_loaders/fs/csv'
import path from 'path'
import { getStoragePath } from '../../../src'
import fs from 'fs'
import { getFileFromStorage } from '../../../src'
class Csv_DocumentLoaders implements INode {
label: string
@@ -75,8 +73,7 @@ class Csv_DocumentLoaders implements INode {
const chatflowid = options.chatflowid
for (const file of files) {
const fileInStorage = path.join(getStoragePath(), chatflowid, file)
const fileData = fs.readFileSync(fileInStorage)
const fileData = await getFileFromStorage(file, chatflowid)
const blob = new Blob([fileData])
const loader = new CSVLoader(blob, columnName.trim().length === 0 ? undefined : columnName.trim())
packages/components/nodes/documentloaders/Docx/Docx.ts
+2 -5
View File
@@ -1,9 +1,7 @@
import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
import { TextSplitter } from 'langchain/text_splitter'
import { DocxLoader } from 'langchain/document_loaders/fs/docx'
import path from 'path'
import { getStoragePath } from '../../../src'
import fs from 'fs'
import { getFileFromStorage } from '../../../src'
class Docx_DocumentLoaders implements INode {
label: string
@@ -66,8 +64,7 @@ class Docx_DocumentLoaders implements INode {
const chatflowid = options.chatflowid
for (const file of files) {
const fileInStorage = path.join(getStoragePath(), chatflowid, file)
const fileData = fs.readFileSync(fileInStorage)
const fileData = await getFileFromStorage(file, chatflowid)
const blob = new Blob([fileData])
const loader = new DocxLoader(blob)
packages/components/nodes/documentloaders/Json/Json.ts
+2 -5
View File
@@ -1,9 +1,7 @@
import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
import { TextSplitter } from 'langchain/text_splitter'
import { JSONLoader } from 'langchain/document_loaders/fs/json'
import { getStoragePath } from '../../../src'
import fs from 'fs'
import path from 'path'
import { getFileFromStorage } from '../../../src'
class Json_DocumentLoaders implements INode {
label: string
@@ -82,8 +80,7 @@ class Json_DocumentLoaders implements INode {
const chatflowid = options.chatflowid
for (const file of files) {
const fileInStorage = path.join(getStoragePath(), chatflowid, file)
const fileData = fs.readFileSync(fileInStorage)
const fileData = await getFileFromStorage(file, chatflowid)
const blob = new Blob([fileData])
const loader = new JSONLoader(blob, pointers.length != 0 ? pointers : undefined)
packages/components/nodes/documentloaders/Jsonlines/Jsonlines.ts
+2 -5
View File
@@ -1,9 +1,7 @@
import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
import { TextSplitter } from 'langchain/text_splitter'
import { JSONLinesLoader } from 'langchain/document_loaders/fs/json'
import { getStoragePath } from '../../../src'
import fs from 'fs'
import path from 'path'
import { getFileFromStorage } from '../../../src'
class Jsonlines_DocumentLoaders implements INode {
label: string
@@ -76,8 +74,7 @@ class Jsonlines_DocumentLoaders implements INode {
const chatflowid = options.chatflowid
for (const file of files) {
const fileInStorage = path.join(getStoragePath(), chatflowid, file)
const fileData = fs.readFileSync(fileInStorage)
const fileData = await getFileFromStorage(file, chatflowid)
const blob = new Blob([fileData])
const loader = new JSONLinesLoader(blob, pointer)
packages/components/nodes/documentloaders/Pdf/Pdf.ts
+2 -5
View File
@@ -1,9 +1,7 @@
import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
import { TextSplitter } from 'langchain/text_splitter'
import { PDFLoader } from 'langchain/document_loaders/fs/pdf'
import { getStoragePath } from '../../../src'
import fs from 'fs'
import path from 'path'
import { getFileFromStorage } from '../../../src'
class Pdf_DocumentLoaders implements INode {
label: string
@@ -92,8 +90,7 @@ class Pdf_DocumentLoaders implements INode {
const chatflowid = options.chatflowid
for (const file of files) {
const fileInStorage = path.join(getStoragePath(), chatflowid, file)
const fileData = fs.readFileSync(fileInStorage)
const fileData = await getFileFromStorage(file, chatflowid)
const bf = Buffer.from(fileData)
await this.extractDocs(usage, bf, legacyBuild, textSplitter, alldocs)
}
packages/components/nodes/documentloaders/Text/Text.ts
+2 -5
View File
@@ -2,9 +2,7 @@ import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from
import { TextSplitter } from 'langchain/text_splitter'
import { TextLoader } from 'langchain/document_loaders/fs/text'
import { Document } from '@langchain/core/documents'
import { getStoragePath, handleEscapeCharacters } from '../../../src'
import fs from 'fs'
import path from 'path'
import { getFileFromStorage, handleEscapeCharacters } from '../../../src'
class Text_DocumentLoaders implements INode {
label: string
@@ -85,8 +83,7 @@ class Text_DocumentLoaders implements INode {
const chatflowid = options.chatflowid
for (const file of files) {
const fileInStorage = path.join(getStoragePath(), chatflowid, file)
const fileData = fs.readFileSync(fileInStorage)
const fileData = await getFileFromStorage(file, chatflowid)
const blob = new Blob([fileData])
const loader = new TextLoader(blob)
packages/components/nodes/tools/OpenAPIToolkit/OpenAPIToolkit.ts
+3 -6
View File
@@ -3,10 +3,7 @@ import { BaseLanguageModel } from '@langchain/core/language_models/base'
import { OpenApiToolkit } from 'langchain/agents'
import { JsonSpec, JsonObject } from './core'
import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
import { getCredentialData, getCredentialParam } from '../../../src'
import { getStoragePath } from '../../../src'
import fs from 'fs'
import path from 'path'
import { getCredentialData, getCredentialParam, getFileFromStorage } from '../../../src'
class OpenAPIToolkit_Tools implements INode {
label: string
@@ -63,9 +60,9 @@ class OpenAPIToolkit_Tools implements INode {
if (yamlFileBase64.startsWith('FILE-STORAGE::')) {
const file = yamlFileBase64.replace('FILE-STORAGE::', '')
const chatflowid = options.chatflowid
const fileInStorage = path.join(getStoragePath(), chatflowid, file)
const fileData = fs.readFileSync(fileInStorage)
const fileData = await getFileFromStorage(file, chatflowid)
const utf8String = fileData.toString('utf-8')
data = load(utf8String) as JsonObject
} else {
const splitDataURI = yamlFileBase64.split(',')
packages/components/nodes/vectorstores/Vectara/Vectara.ts
+2 -5
View File
@@ -11,9 +11,7 @@ import { Document } from '@langchain/core/documents'
import { Embeddings } from '@langchain/core/embeddings'
import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface'
import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
import { getStoragePath } from '../../../src'
import fs from 'fs'
import path from 'path'
import { getFileFromStorage } from '../../../src'
class Vectara_VectorStores implements INode {
label: string
@@ -197,8 +195,7 @@ class Vectara_VectorStores implements INode {
const chatflowid = options.chatflowid
for (const file of files) {
const fileInStorage = path.join(getStoragePath(), chatflowid, file)
const fileData = fs.readFileSync(fileInStorage)
const fileData = await getFileFromStorage(file, chatflowid)
const blob = new Blob([fileData])
vectaraFiles.push({ blob: blob, fileName: getFileName(file) })
}
packages/components/nodes/vectorstores/Vectara/Vectara_Upload.ts
+2 -5
View File
@@ -1,9 +1,7 @@
import { VectaraStore, VectaraLibArgs, VectaraFilter, VectaraContextConfig, VectaraFile } from '@langchain/community/vectorstores/vectara'
import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
import path from 'path'
import { getStoragePath } from '../../../src'
import fs from 'fs'
import { getFileFromStorage } from '../../../src'
class VectaraUpload_VectorStores implements INode {
label: string
@@ -144,8 +142,7 @@ class VectaraUpload_VectorStores implements INode {
const chatflowid = options.chatflowid
for (const file of files) {
const fileInStorage = path.join(getStoragePath(), chatflowid, file)
const fileData = fs.readFileSync(fileInStorage)
const fileData = await getFileFromStorage(file, chatflowid)
const blob = new Blob([fileData])
vectaraFiles.push({ blob: blob, fileName: getFileName(file) })
}