farcasclaudiu/Flowise
1
0
Fork 0
mirror of https://github.com/farcasclaudiu/Flowise.git synced 2026-06-28 15:00:57 +03:00
Code Issues Packages Projects Releases Wiki Activity

Chore/refractor (#4454)

Browse Source 
* markdown files and env examples cleanup

* components update

* update jsonlines description

* server refractor

* update telemetry

* add execute custom node

* add ui refractor

* add username and password authenticate

* correctly retrieve past images in agentflowv2

* disable e2e temporarily

* add existing username and password authenticate

* update migration to default workspace

* update todo

* blob storage migrating

* throw error on agent tool call error

* add missing execution import

* add referral

* chore: add error message when importData is undefined

* migrate api keys to db

* fix: data too long for column executionData

* migrate api keys from json to db at init

* add info on account setup

* update docstore missing fields

---------

Co-authored-by: chungyau97 <chungyau97@gmail.com>
This commit is contained in:
Henry Heng
2025-05-27 14:29:42 +08:00
committed by GitHub
parent e35a126b46
commit 5a37227d14
560 changed files with 62127 additions and 4100 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/components/nodes/documentloaders/Cheerio/Cheerio.ts
+12 -6
View File
@@ -123,6 +123,7 @@ class Cheerio_DocumentLoaders implements INode {
const selectedLinks = nodeData.inputs?.selectedLinks as string[]
let limit = parseInt(nodeData.inputs?.limit as string)
const output = nodeData.outputs?.output as string
const orgId = options.orgId
const _omitMetadataKeys = nodeData.inputs?.omitMetadataKeys as string
@@ -149,7 +150,8 @@ class Cheerio_DocumentLoaders implements INode {
try {
let docs: IDocument[] = []
if (url.endsWith('.pdf')) {
if (process.env.DEBUG === 'true') options.logger.info(`CheerioWebBaseLoader does not support PDF files: ${url}`)
if (process.env.DEBUG === 'true')
options.logger.info(`[${orgId}]: CheerioWebBaseLoader does not support PDF files: ${url}`)
return docs
}
const loader = new CheerioWebBaseLoader(url, params)
@@ -161,7 +163,8 @@ class Cheerio_DocumentLoaders implements INode {
}
return docs
} catch (err) {
if (process.env.DEBUG === 'true') options.logger.error(`error in CheerioWebBaseLoader: ${err.message}, on page: ${url}`)
if (process.env.DEBUG === 'true')
options.logger.error(`[${orgId}]: Error in CheerioWebBaseLoader: ${err.message}, on page: ${url}`)
return []
}
}
@@ -169,7 +172,7 @@ class Cheerio_DocumentLoaders implements INode {
let docs: IDocument[] = []
if (relativeLinksMethod) {
if (process.env.DEBUG === 'true') options.logger.info(`Start ${relativeLinksMethod}`)
if (process.env.DEBUG === 'true') options.logger.info(`[${orgId}]: Start CheerioWebBaseLoader ${relativeLinksMethod}`)
// if limit is 0 we don't want it to default to 10 so we check explicitly for null or undefined
// so when limit is 0 we can fetch all the links
if (limit === null || limit === undefined) limit = 10
@@ -180,15 +183,18 @@ class Cheerio_DocumentLoaders implements INode {
: relativeLinksMethod === 'webCrawl'
? await webCrawl(url, limit)
: await xmlScrape(url, limit)
if (process.env.DEBUG === 'true') options.logger.info(`pages: ${JSON.stringify(pages)}, length: ${pages.length}`)
if (process.env.DEBUG === 'true')
options.logger.info(`[${orgId}]: CheerioWebBaseLoader pages: ${JSON.stringify(pages)}, length: ${pages.length}`)
if (!pages || pages.length === 0) throw new Error('No relative links found')
for (const page of pages) {
docs.push(...(await cheerioLoader(page)))
}
if (process.env.DEBUG === 'true') options.logger.info(`Finish ${relativeLinksMethod}`)
if (process.env.DEBUG === 'true') options.logger.info(`[${orgId}]: Finish CheerioWebBaseLoader ${relativeLinksMethod}`)
} else if (selectedLinks && selectedLinks.length > 0) {
if (process.env.DEBUG === 'true')
options.logger.info(`pages: ${JSON.stringify(selectedLinks)}, length: ${selectedLinks.length}`)
options.logger.info(
`[${orgId}]: CheerioWebBaseLoader pages: ${JSON.stringify(selectedLinks)}, length: ${selectedLinks.length}`
)
for (const page of selectedLinks.slice(0, limit)) {
docs.push(...(await cheerioLoader(page)))
}
packages/components/nodes/documentloaders/Csv/Csv.ts
+4 -3
View File
@@ -107,9 +107,9 @@ class Csv_DocumentLoaders implements INode {
return { files, fromStorage }
}
async getFileData(file: string, { chatflowid }: { chatflowid: string }, fromStorage?: boolean) {
async getFileData(file: string, { orgId, chatflowid }: { orgId: string; chatflowid: string }, fromStorage?: boolean) {
if (fromStorage) {
return getFileFromStorage(file, chatflowid)
return getFileFromStorage(file, orgId, chatflowid)
} else {
const splitDataURI = file.split(',')
splitDataURI.pop()
@@ -126,6 +126,7 @@ class Csv_DocumentLoaders implements INode {
let docs: IDocument[] = []
const orgId = options.orgId
const chatflowid = options.chatflowid
const { files, fromStorage } = this.getFiles(nodeData)
@@ -133,7 +134,7 @@ class Csv_DocumentLoaders implements INode {
for (const file of files) {
if (!file) continue
const fileData = await this.getFileData(file, { chatflowid }, fromStorage)
const fileData = await this.getFileData(file, { orgId, chatflowid }, fromStorage)
const blob = new Blob([fileData])
const loader = new CSVLoader(blob, columnName.trim().length === 0 ? undefined : columnName.trim())
packages/components/nodes/documentloaders/CustomDocumentLoader/CustomDocumentLoader.ts
+1 -1
View File
@@ -72,7 +72,7 @@ class CustomDocumentLoader_DocumentLoaders implements INode {
const appDataSource = options.appDataSource as DataSource
const databaseEntities = options.databaseEntities as IDatabaseEntity
const variables = await getVars(appDataSource, databaseEntities, nodeData)
const variables = await getVars(appDataSource, databaseEntities, nodeData, options)
const flow = {
chatflowId: options.chatflowid,
sessionId: options.sessionId,
packages/components/nodes/documentloaders/DocumentStore/DocStoreLoader.ts
+2 -1
View File
@@ -60,7 +60,8 @@ class DocStore_DocumentLoaders implements INode {
return returnData
}
const stores = await appDataSource.getRepository(databaseEntities['DocumentStore']).find()
const searchOptions = options.searchOptions || {}
const stores = await appDataSource.getRepository(databaseEntities['DocumentStore']).findBy(searchOptions)
for (const store of stores) {
if (store.status === 'SYNC') {
const obj = {
packages/components/nodes/documentloaders/Docx/Docx.ts
+2 -1
View File
@@ -96,11 +96,12 @@ class Docx_DocumentLoaders implements INode {
} else {
files = [fileName]
}
const orgId = options.orgId
const chatflowid = options.chatflowid
for (const file of files) {
if (!file) continue
const fileData = await getFileFromStorage(file, chatflowid)
const fileData = await getFileFromStorage(file, orgId, chatflowid)
const blob = new Blob([fileData])
const loader = new DocxLoader(blob)
packages/components/nodes/documentloaders/Epub/Epub.ts
+2 -1
View File
@@ -118,10 +118,11 @@ class Epub_DocumentLoaders implements INode {
files = fileName.startsWith('[') && fileName.endsWith(']') ? JSON.parse(fileName) : [fileName]
const chatflowid = options.chatflowid
const orgId = options.orgId
for (const file of files) {
if (!file) continue
const fileData = await getFileFromStorage(file, chatflowid)
const fileData = await getFileFromStorage(file, orgId, chatflowid)
const tempFilePath = path.join(tempDir, `${Date.now()}_${file}`)
fs.writeFileSync(tempFilePath, fileData)
await this.extractDocs(usage, tempFilePath, textSplitter, docs)
packages/components/nodes/documentloaders/File/File.ts
+3 -2
View File
@@ -144,6 +144,7 @@ class File_DocumentLoaders implements INode {
} else {
files = [fileName]
}
const orgId = options.orgId
const chatflowid = options.chatflowid
// specific to createAttachment to get files from chatId
@@ -151,14 +152,14 @@ class File_DocumentLoaders implements INode {
if (retrieveAttachmentChatId) {
for (const file of files) {
if (!file) continue
const fileData = await getFileFromStorage(file, chatflowid, options.chatId)
const fileData = await getFileFromStorage(file, orgId, chatflowid, options.chatId)
const blob = new Blob([fileData])
fileBlobs.push({ blob, ext: file.split('.').pop() || '' })
}
} else {
for (const file of files) {
if (!file) continue
const fileData = await getFileFromStorage(file, chatflowid)
const fileData = await getFileFromStorage(file, orgId, chatflowid)
const blob = new Blob([fileData])
fileBlobs.push({ blob, ext: file.split('.').pop() || '' })
}
packages/components/nodes/documentloaders/Json/Json.ts
+2 -1
View File
@@ -146,11 +146,12 @@ class Json_DocumentLoaders implements INode {
} else {
files = [fileName]
}
const orgId = options.orgId
const chatflowid = options.chatflowid
for (const file of files) {
if (!file) continue
const fileData = await getFileFromStorage(file, chatflowid)
const fileData = await getFileFromStorage(file, orgId, chatflowid)
const blob = new Blob([fileData])
const loader = new JSONLoader(blob, pointers.length != 0 ? pointers : undefined, metadata)
packages/components/nodes/documentloaders/Jsonlines/Jsonlines.ts
+2 -1
View File
@@ -135,11 +135,12 @@ class Jsonlines_DocumentLoaders implements INode {
} else {
files = [fileName]
}
const orgId = options.orgId
const chatflowid = options.chatflowid
for (const file of files) {
if (!file) continue
const fileData = await getFileFromStorage(file, chatflowid)
const fileData = await getFileFromStorage(file, orgId, chatflowid)
const blob = new Blob([fileData])
const loader = new JSONLinesLoader(blob, pointer, metadata)
packages/components/nodes/documentloaders/Pdf/Pdf.ts
+2 -1
View File
@@ -122,11 +122,12 @@ class Pdf_DocumentLoaders implements INode {
} else {
files = [fileName]
}
const orgId = options.orgId
const chatflowid = options.chatflowid
for (const file of files) {
if (!file) continue
const fileData = await getFileFromStorage(file, chatflowid)
const fileData = await getFileFromStorage(file, orgId, chatflowid)
const bf = Buffer.from(fileData)
await this.extractDocs(usage, bf, legacyBuild, textSplitter, docs)
}
packages/components/nodes/documentloaders/Playwright/Playwright.ts
+10 -5
View File
@@ -159,6 +159,7 @@ class Playwright_DocumentLoaders implements INode {
let waitForSelector = nodeData.inputs?.waitForSelector as string
const _omitMetadataKeys = nodeData.inputs?.omitMetadataKeys as string
const output = nodeData.outputs?.output as string
const orgId = options.orgId
let omitMetadataKeys: string[] = []
if (_omitMetadataKeys) {
@@ -202,13 +203,14 @@ class Playwright_DocumentLoaders implements INode {
}
return docs
} catch (err) {
if (process.env.DEBUG === 'true') options.logger.error(`error in PlaywrightWebBaseLoader: ${err.message}, on page: ${url}`)
if (process.env.DEBUG === 'true')
options.logger.error(`[${orgId}]: Error in PlaywrightWebBaseLoader: ${err.message}, on page: ${url}`)
}
}
let docs: IDocument[] = []
if (relativeLinksMethod) {
if (process.env.DEBUG === 'true') options.logger.info(`Start ${relativeLinksMethod}`)
if (process.env.DEBUG === 'true') options.logger.info(`[${orgId}]: Start PlaywrightWebBaseLoader ${relativeLinksMethod}`)
// if limit is 0 we don't want it to default to 10 so we check explicitly for null or undefined
// so when limit is 0 we can fetch all the links
if (limit === null || limit === undefined) limit = 10
@@ -219,15 +221,18 @@ class Playwright_DocumentLoaders implements INode {
: relativeLinksMethod === 'webCrawl'
? await webCrawl(url, limit)
: await xmlScrape(url, limit)
if (process.env.DEBUG === 'true') options.logger.info(`pages: ${JSON.stringify(pages)}, length: ${pages.length}`)
if (process.env.DEBUG === 'true')
options.logger.info(`[${orgId}]: PlaywrightWebBaseLoader pages: ${JSON.stringify(pages)}, length: ${pages.length}`)
if (!pages || pages.length === 0) throw new Error('No relative links found')
for (const page of pages) {
docs.push(...(await playwrightLoader(page)))
}
if (process.env.DEBUG === 'true') options.logger.info(`Finish ${relativeLinksMethod}`)
if (process.env.DEBUG === 'true') options.logger.info(`[${orgId}]: Finish PlaywrightWebBaseLoader ${relativeLinksMethod}`)
} else if (selectedLinks && selectedLinks.length > 0) {
if (process.env.DEBUG === 'true')
options.logger.info(`pages: ${JSON.stringify(selectedLinks)}, length: ${selectedLinks.length}`)
options.logger.info(
`[${orgId}]: PlaywrightWebBaseLoader pages: ${JSON.stringify(selectedLinks)}, length: ${selectedLinks.length}`
)
for (const page of selectedLinks.slice(0, limit)) {
docs.push(...(await playwrightLoader(page)))
}
packages/components/nodes/documentloaders/Puppeteer/Puppeteer.ts
+10 -5
View File
@@ -155,6 +155,7 @@ class Puppeteer_DocumentLoaders implements INode {
let waitForSelector = nodeData.inputs?.waitForSelector as string
const _omitMetadataKeys = nodeData.inputs?.omitMetadataKeys as string
const output = nodeData.outputs?.output as string
const orgId = options.orgId
let omitMetadataKeys: string[] = []
if (_omitMetadataKeys) {
@@ -198,13 +199,14 @@ class Puppeteer_DocumentLoaders implements INode {
}
return docs
} catch (err) {
if (process.env.DEBUG === 'true') options.logger.error(`error in PuppeteerWebBaseLoader: ${err.message}, on page: ${url}`)
if (process.env.DEBUG === 'true')
options.logger.error(`[${orgId}]: Error in PuppeteerWebBaseLoader: ${err.message}, on page: ${url}`)
}
}
let docs: IDocument[] = []
if (relativeLinksMethod) {
if (process.env.DEBUG === 'true') options.logger.info(`Start ${relativeLinksMethod}`)
if (process.env.DEBUG === 'true') options.logger.info(`[${orgId}]: Start PuppeteerWebBaseLoader ${relativeLinksMethod}`)
// if limit is 0 we don't want it to default to 10 so we check explicitly for null or undefined
// so when limit is 0 we can fetch all the links
if (limit === null || limit === undefined) limit = 10
@@ -215,15 +217,18 @@ class Puppeteer_DocumentLoaders implements INode {
: relativeLinksMethod === 'webCrawl'
? await webCrawl(url, limit)
: await xmlScrape(url, limit)
if (process.env.DEBUG === 'true') options.logger.info(`pages: ${JSON.stringify(pages)}, length: ${pages.length}`)
if (process.env.DEBUG === 'true')
options.logger.info(`[${orgId}]: PuppeteerWebBaseLoader pages: ${JSON.stringify(pages)}, length: ${pages.length}`)
if (!pages || pages.length === 0) throw new Error('No relative links found')
for (const page of pages) {
docs.push(...(await puppeteerLoader(page)))
}
if (process.env.DEBUG === 'true') options.logger.info(`Finish ${relativeLinksMethod}`)
if (process.env.DEBUG === 'true') options.logger.info(`[${orgId}]: Finish PuppeteerWebBaseLoader ${relativeLinksMethod}`)
} else if (selectedLinks && selectedLinks.length > 0) {
if (process.env.DEBUG === 'true')
options.logger.info(`pages: ${JSON.stringify(selectedLinks)}, length: ${selectedLinks.length}`)
options.logger.info(
`[${orgId}]: PuppeteerWebBaseLoader pages: ${JSON.stringify(selectedLinks)}, length: ${selectedLinks.length}`
)
for (const page of selectedLinks.slice(0, limit)) {
docs.push(...(await puppeteerLoader(page)))
}
packages/components/nodes/documentloaders/S3File/README.md
+4 -4
View File
@@ -4,10 +4,10 @@ DS File Loarder integration for Flowise
## 🌱 Env Variables
| Variable | Description | Type | Default |
| ---------------------------- | ----------------------------------------------------------------------------------------------- | ------------------------------------------------ | ----------------------------------- |
| UNSTRUCTURED_API_URL | Default `unstructuredApiUrl` for S3 File Loader | String | http://localhost:8000/general/v0/general |
| Variable | Description | Type | Default |
| -------------------- | ----------------------------------------------- | ------ | ---------------------------------------- |
| UNSTRUCTURED_API_URL | Default `unstructuredApiUrl` for S3 File Loader | String | http://localhost:8000/general/v0/general |
## License
Source code in this repository is made available under the [Apache License Version 2.0](https://github.com/FlowiseAI/Flowise/blob/master/LICENSE.md).
Source code in this repository is made available under the [Apache License Version 2.0](https://github.com/FlowiseAI/Flowise/blob/master/LICENSE.md).
packages/components/nodes/documentloaders/Text/Text.ts
+2 -1
View File
@@ -98,11 +98,12 @@ class Text_DocumentLoaders implements INode {
} else {
files = [fileName]
}
const orgId = options.orgId
const chatflowid = options.chatflowid
for (const file of files) {
if (!file) continue
const fileData = await getFileFromStorage(file, chatflowid)
const fileData = await getFileFromStorage(file, orgId, chatflowid)
const blob = new Blob([fileData])
const loader = new TextLoader(blob)
packages/components/nodes/documentloaders/Unstructured/README.md
+4 -4
View File
@@ -4,10 +4,10 @@ Unstructured File Loader integration for Flowise
## 🌱 Env Variables
| Variable | Description | Type | Default |
| ---------------------------- | ----------------------------------------------------------------------------------------------- | ------------------------------------------------ | ----------------------------------- |
| UNSTRUCTURED_API_URL | Default `apiUrl` for Unstructured File/Floder Loader | String | http://localhost:8000/general/v0/general |
| Variable | Description | Type | Default |
| -------------------- | ---------------------------------------------------- | ------ | ---------------------------------------- |
| UNSTRUCTURED_API_URL | Default `apiUrl` for Unstructured File/Floder Loader | String | http://localhost:8000/general/v0/general |
## License
Source code in this repository is made available under the [Apache License Version 2.0](https://github.com/FlowiseAI/Flowise/blob/master/LICENSE.md).
Source code in this repository is made available under the [Apache License Version 2.0](https://github.com/FlowiseAI/Flowise/blob/master/LICENSE.md).
packages/components/nodes/documentloaders/Unstructured/UnstructuredFile.ts
+2 -1
View File
@@ -532,11 +532,12 @@ class UnstructuredFile_DocumentLoaders implements INode {
} else {
files = [fileName]
}
const orgId = options.orgId
const chatflowid = options.chatflowid
for (const file of files) {
if (!file) continue
const fileData = await getFileFromStorage(file, chatflowid)
const fileData = await getFileFromStorage(file, orgId, chatflowid)
const loaderDocs = await loader.loadAndSplitBuffer(fileData, file)
docs.push(...loaderDocs)
}