farcasclaudiu/Flowise
1
0
Fork 0
mirror of https://github.com/farcasclaudiu/Flowise.git synced 2026-06-28 19:00:59 +03:00
Code Issues Packages Projects Releases Wiki Activity

Feature/externalize files from chatflow - do not save as base64 (#1976)

Browse Source 
* initial commit. Externalizing the file base64 string from flowData

* csv - docloader - Externalizing the file base64 string from flowData

* csv - docloader - Externalizing the file base64 string from flowData

* DocX - docloader - Externalizing the file base64 string from flowData

* Json - docloader - Externalizing the file base64 string from flowData

* Jsonlines - docloader - Externalizing the file base64 string from flowData

* PDF - docloader - Externalizing the file base64 string from flowData

* Vectara - vector store - Externalizing the file base64 string from flowData

* OpenAPIToolkit - tools - Externalizing the file base64 string from flowData

* OpenAPIChain - chain - Externalizing the file base64 string from flowData

* lint fixes

* datasource enabled - initial commit

* CSVAgent - agents - Externalizing the file base64 string from flowData

* Externalizing the file base64 string from flowData

* Externalizing the file base64 string from flowData

* add pnpm-lock.yaml

* update filerepository to add try catch

* Rename FileRepository.ts to fileRepository.ts

---------

Co-authored-by: Henry <hzj94@hotmail.com>
Co-authored-by: Henry Heng <henryheng@flowiseai.com>
This commit is contained in:
Vinod Kiran
2024-04-04 21:41:06 +05:30
committed by GitHub
parent eed7de6df5
commit 658fa3984e
16 changed files with 593 additions and 194 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/components/nodes/agents/CSVAgent/CSVAgent.ts
+29 -11
View File
@@ -7,6 +7,9 @@ import { getBaseClasses } from '../../../src/utils'
import { LoadPyodide, finalSystemPrompt, systemPrompt } from './core'
import { checkInputs, Moderation } from '../../moderation/Moderation'
import { formatResponse } from '../../outputparsers/OutputParserHelpers'
import path from 'path'
import { getStoragePath } from '../../../src'
import fs from 'fs'
class CSV_Agents implements INode {
label: string
@@ -88,19 +91,34 @@ class CSV_Agents implements INode {
const callbacks = await additionalCallbacks(nodeData, options)
let files: string[] = []
if (csvFileBase64.startsWith('[') && csvFileBase64.endsWith(']')) {
files = JSON.parse(csvFileBase64)
} else {
files = [csvFileBase64]
}
let base64String = ''
for (const file of files) {
const splitDataURI = file.split(',')
splitDataURI.pop()
base64String += splitDataURI.pop() ?? ''
if (csvFileBase64.startsWith('FILE-STORAGE::')) {
const fileName = csvFileBase64.replace('FILE-STORAGE::', '')
if (fileName.startsWith('[') && fileName.endsWith(']')) {
files = JSON.parse(fileName)
} else {
files = [fileName]
}
const chatflowid = options.chatflowid
for (const file of files) {
const fileInStorage = path.join(getStoragePath(), chatflowid, file)
const fileData = fs.readFileSync(fileInStorage)
base64String += fileData.toString('base64')
}
} else {
if (csvFileBase64.startsWith('[') && csvFileBase64.endsWith(']')) {
files = JSON.parse(csvFileBase64)
} else {
files = [csvFileBase64]
}
for (const file of files) {
const splitDataURI = file.split(',')
splitDataURI.pop()
base64String += splitDataURI.pop() ?? ''
}
}
const pyodide = await LoadPyodide()