farcasclaudiu/Flowise
1
0
Fork 0
mirror of https://github.com/farcasclaudiu/Flowise.git synced 2026-06-29 01:01:11 +03:00
Code Issues Packages Projects Releases Wiki Activity

FIX #2617 Cherio Web Crawler doesn't work with large sites (#2678)

Browse Source 
* FIX #2617 Big sites scan error

* FIX #2617 Big sites scan error - review fix

---------

Co-authored-by: Ahmed Osman <ahmed.osman@evolpe.pl>
This commit is contained in:
Ahmed Osman
2024-07-05 12:34:47 +02:00
committed by GitHub
parent b1e38783e4
commit 90558ca688
1 changed files with 6 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/components/nodes/documentloaders/Cheerio/Cheerio.ts
+6 -1
View File
@@ -131,7 +131,11 @@ class Cheerio_DocumentLoaders implements INode {
async function cheerioLoader(url: string): Promise<any> { async function cheerioLoader(url: string): Promise<any> {
try { try {
let docs = [] let docs: IDocument[] = []
if (url.endsWith('.pdf')) {
if (process.env.DEBUG === 'true') options.logger.info(`CheerioWebBaseLoader does not support PDF files: ${url}`)
return docs
}
const loader = new CheerioWebBaseLoader(url, params) const loader = new CheerioWebBaseLoader(url, params)
if (textSplitter) { if (textSplitter) {
docs = await loader.loadAndSplit(textSplitter) docs = await loader.loadAndSplit(textSplitter)
@@ -141,6 +145,7 @@ class Cheerio_DocumentLoaders implements INode {
return docs return docs
} catch (err) { } catch (err) {
if (process.env.DEBUG === 'true') options.logger.error(`error in CheerioWebBaseLoader: ${err.message}, on page: ${url}`) if (process.env.DEBUG === 'true') options.logger.error(`error in CheerioWebBaseLoader: ${err.message}, on page: ${url}`)
return []
} }
} }