farcasclaudiu/Flowise
1
0
Fork 0
mirror of https://github.com/farcasclaudiu/Flowise.git synced 2026-06-28 19:00:59 +03:00
Code Issues Packages Projects Releases Wiki Activity

Chore/refractor (#4454)

Browse Source 
* markdown files and env examples cleanup

* components update

* update jsonlines description

* server refractor

* update telemetry

* add execute custom node

* add ui refractor

* add username and password authenticate

* correctly retrieve past images in agentflowv2

* disable e2e temporarily

* add existing username and password authenticate

* update migration to default workspace

* update todo

* blob storage migrating

* throw error on agent tool call error

* add missing execution import

* add referral

* chore: add error message when importData is undefined

* migrate api keys to db

* fix: data too long for column executionData

* migrate api keys from json to db at init

* add info on account setup

* update docstore missing fields

---------

Co-authored-by: chungyau97 <chungyau97@gmail.com>
This commit is contained in:
Henry Heng
2025-05-27 14:29:42 +08:00
committed by GitHub
parent e35a126b46
commit 5a37227d14
560 changed files with 62127 additions and 4100 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/components/nodes/documentloaders/Cheerio/Cheerio.ts
+12 -6
View File
@@ -123,6 +123,7 @@ class Cheerio_DocumentLoaders implements INode {
const selectedLinks = nodeData.inputs?.selectedLinks as string[]
let limit = parseInt(nodeData.inputs?.limit as string)
const output = nodeData.outputs?.output as string
const orgId = options.orgId
const _omitMetadataKeys = nodeData.inputs?.omitMetadataKeys as string
@@ -149,7 +150,8 @@ class Cheerio_DocumentLoaders implements INode {
try {
let docs: IDocument[] = []
if (url.endsWith('.pdf')) {
if (process.env.DEBUG === 'true') options.logger.info(`CheerioWebBaseLoader does not support PDF files: ${url}`)
if (process.env.DEBUG === 'true')
options.logger.info(`[${orgId}]: CheerioWebBaseLoader does not support PDF files: ${url}`)
return docs
}
const loader = new CheerioWebBaseLoader(url, params)
@@ -161,7 +163,8 @@ class Cheerio_DocumentLoaders implements INode {
}
return docs
} catch (err) {
if (process.env.DEBUG === 'true') options.logger.error(`error in CheerioWebBaseLoader: ${err.message}, on page: ${url}`)
if (process.env.DEBUG === 'true')
options.logger.error(`[${orgId}]: Error in CheerioWebBaseLoader: ${err.message}, on page: ${url}`)
return []
}
}
@@ -169,7 +172,7 @@ class Cheerio_DocumentLoaders implements INode {
let docs: IDocument[] = []
if (relativeLinksMethod) {
if (process.env.DEBUG === 'true') options.logger.info(`Start ${relativeLinksMethod}`)
if (process.env.DEBUG === 'true') options.logger.info(`[${orgId}]: Start CheerioWebBaseLoader ${relativeLinksMethod}`)
// if limit is 0 we don't want it to default to 10 so we check explicitly for null or undefined
// so when limit is 0 we can fetch all the links
if (limit === null || limit === undefined) limit = 10
@@ -180,15 +183,18 @@ class Cheerio_DocumentLoaders implements INode {
: relativeLinksMethod === 'webCrawl'
? await webCrawl(url, limit)
: await xmlScrape(url, limit)
if (process.env.DEBUG === 'true') options.logger.info(`pages: ${JSON.stringify(pages)}, length: ${pages.length}`)
if (process.env.DEBUG === 'true')
options.logger.info(`[${orgId}]: CheerioWebBaseLoader pages: ${JSON.stringify(pages)}, length: ${pages.length}`)
if (!pages || pages.length === 0) throw new Error('No relative links found')
for (const page of pages) {
docs.push(...(await cheerioLoader(page)))
}
if (process.env.DEBUG === 'true') options.logger.info(`Finish ${relativeLinksMethod}`)
if (process.env.DEBUG === 'true') options.logger.info(`[${orgId}]: Finish CheerioWebBaseLoader ${relativeLinksMethod}`)
} else if (selectedLinks && selectedLinks.length > 0) {
if (process.env.DEBUG === 'true')
options.logger.info(`pages: ${JSON.stringify(selectedLinks)}, length: ${selectedLinks.length}`)
options.logger.info(
`[${orgId}]: CheerioWebBaseLoader pages: ${JSON.stringify(selectedLinks)}, length: ${selectedLinks.length}`
)
for (const page of selectedLinks.slice(0, limit)) {
docs.push(...(await cheerioLoader(page)))
}