From b9b0c9d227a9260b7e80a4272344a15801f25da4 Mon Sep 17 00:00:00 2001 From: Henry Heng Date: Tue, 2 Apr 2024 11:14:04 +0100 Subject: [PATCH] Bugfix/Web Scraper Limit (#2083) fix when limit set to 0, selectedLinks sliced to become empty --- packages/components/nodes/documentloaders/Cheerio/Cheerio.ts | 2 +- .../components/nodes/documentloaders/Playwright/Playwright.ts | 2 +- .../components/nodes/documentloaders/Puppeteer/Puppeteer.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/components/nodes/documentloaders/Cheerio/Cheerio.ts b/packages/components/nodes/documentloaders/Cheerio/Cheerio.ts index 48ae85bc..8a1b763d 100644 --- a/packages/components/nodes/documentloaders/Cheerio/Cheerio.ts +++ b/packages/components/nodes/documentloaders/Cheerio/Cheerio.ts @@ -132,7 +132,7 @@ class Cheerio_DocumentLoaders implements INode { else if (limit < 0) throw new Error('Limit cannot be less than 0') const pages: string[] = selectedLinks && selectedLinks.length > 0 - ? selectedLinks.slice(0, limit) + ? selectedLinks.slice(0, limit === 0 ? undefined : limit) : relativeLinksMethod === 'webCrawl' ? await webCrawl(url, limit) : await xmlScrape(url, limit) diff --git a/packages/components/nodes/documentloaders/Playwright/Playwright.ts b/packages/components/nodes/documentloaders/Playwright/Playwright.ts index 55fa9608..990bde10 100644 --- a/packages/components/nodes/documentloaders/Playwright/Playwright.ts +++ b/packages/components/nodes/documentloaders/Playwright/Playwright.ts @@ -173,7 +173,7 @@ class Playwright_DocumentLoaders implements INode { else if (limit < 0) throw new Error('Limit cannot be less than 0') const pages: string[] = selectedLinks && selectedLinks.length > 0 - ? selectedLinks.slice(0, limit) + ? selectedLinks.slice(0, limit === 0 ? undefined : limit) : relativeLinksMethod === 'webCrawl' ? await webCrawl(url, limit) : await xmlScrape(url, limit) diff --git a/packages/components/nodes/documentloaders/Puppeteer/Puppeteer.ts b/packages/components/nodes/documentloaders/Puppeteer/Puppeteer.ts index 90b5a277..16c75332 100644 --- a/packages/components/nodes/documentloaders/Puppeteer/Puppeteer.ts +++ b/packages/components/nodes/documentloaders/Puppeteer/Puppeteer.ts @@ -174,7 +174,7 @@ class Puppeteer_DocumentLoaders implements INode { else if (limit < 0) throw new Error('Limit cannot be less than 0') const pages: string[] = selectedLinks && selectedLinks.length > 0 - ? selectedLinks.slice(0, limit) + ? selectedLinks.slice(0, limit === 0 ? undefined : limit) : relativeLinksMethod === 'webCrawl' ? await webCrawl(url, limit) : await xmlScrape(url, limit)