Feature/Add Cheerio Web Scraper node (#4337)

* feat: add Free Web Scraper tool

* fix: remove unused variable 'finalCount'

* feat(tools): Add configurable timeout to handle edge scenarios

* add sitemap mode and rename Cheerio Web Scraper node

* refactor: simplify tool description for broader LLM compatibility

packages/components/nodes/tools/FreeWebScraper/CheerioWebScraper.ts

@@ -0,0 +1,434 @@
+import { INode, INodeParams, INodeData, ICommonObject } from '../../../src/Interface'
+import { getBaseClasses } from '../../../src/utils'
+import { Tool } from '@langchain/core/tools'
+import fetch from 'node-fetch'
+import * as cheerio from 'cheerio'
+import { URL } from 'url'
+import { xmlScrape } from '../../../src/utils'
+
+interface ScrapedPageData {
+    url: string
+    title: string
+    description: string
+    body_text: string
+    error?: string
+}
+
+class WebScraperRecursiveTool extends Tool {
+    name = 'cheerio_web_scraper'
+    description = `Scrapes web pages recursively or via default sitemap. Extracts title, description, and paragraph text. Input should be a single URL string. Returns a JSON string array of scraped page data objects.`
+
+    private maxDepth: number
+    private maxPages: number | null
+    private timeoutMs: number
+    private useSitemap: boolean
+    private visitedUrls: Set<string>
+    private scrapedPagesCount: number
+
+    constructor(maxDepth: number = 1, maxPages: number | null = 10, timeoutMs: number = 60000, useSitemap: boolean = false) {
+        super()
+
+        this.maxDepth = Math.max(1, maxDepth)
+        this.maxPages = maxPages !== null && maxPages > 0 ? maxPages : null
+        this.timeoutMs = timeoutMs > 0 ? timeoutMs : 60000
+        this.useSitemap = useSitemap
+        this.visitedUrls = new Set<string>()
+        this.scrapedPagesCount = 0
+
+        let desc = ''
+        if (this.useSitemap) {
+            desc = `Scrapes URLs listed in the detected default sitemap (/sitemap.xml)`
+            if (this.maxPages !== null) {
+                desc += ` up to ${this.maxPages} pages`
+            }
+            desc += `, with a ${
+                this.timeoutMs / 1000
+            }-second timeout per page. Falls back to Recursive Link Following if sitemap is not found or empty.`
+        } else {
+            desc = `Recursively scrapes web pages starting from a given URL`
+            if (this.maxDepth > 0) {
+                desc += ` up to ${this.maxDepth} level(s) deep`
+            }
+            if (this.maxPages !== null) {
+                desc += ` or until ${this.maxPages} pages are scraped`
+            }
+            desc += `, with a ${this.timeoutMs / 1000}-second timeout per page, whichever comes first.`
+        }
+        desc += ` Extracts title, description, and paragraph text. Input should be a single URL string. Returns a JSON string array of scraped page data.`
+        this.description = desc
+    }
+
+    private async scrapeSingleUrl(url: string): Promise<Omit<ScrapedPageData, 'url'> & { foundLinks: string[] }> {
+        try {
+            const response = await fetch(url, { timeout: this.timeoutMs, redirect: 'follow', follow: 5 })
+            if (!response.ok) {
+                const errorText = await response.text()
+                return {
+                    title: '',
+                    description: '',
+                    body_text: '',
+                    foundLinks: [],
+                    error: `HTTP Error: ${response.status} ${response.statusText}. ${errorText}`
+                }
+            }
+            const contentType = response.headers.get('content-type')
+
+            if (contentType === null) {
+                return {
+                    title: '',
+                    description: '',
+                    body_text: '',
+                    foundLinks: [],
+                    error: `Skipped content due to missing Content-Type header`
+                }
+            }
+
+            if (!contentType.includes('text/html') && url !== this.visitedUrls.values().next().value) {
+                if (!contentType.includes('text/xml') && !contentType.includes('application/xml')) {
+                    return {
+                        title: '',
+                        description: '',
+                        body_text: '',
+                        foundLinks: [],
+                        error: `Skipped non-HTML/XML content (Content-Type: ${contentType})`
+                    }
+                }
+
+                if (!contentType.includes('text/html')) {
+                    return {
+                        title: '',
+                        description: '',
+                        body_text: '',
+                        foundLinks: [],
+                        error: `Skipped non-HTML content (Content-Type: ${contentType})`
+                    }
+                }
+            }
+
+            const html = await response.text()
+            const $ = cheerio.load(html)
+            const title = $('title').first().text() || 'No title found'
+            let description =
+                $('meta[name="description"]').attr('content') ||
+                $('meta[property="og:description"]').attr('content') ||
+                $('meta[name="twitter:description"]').attr('content') ||
+                'No description found'
+            const paragraphs: string[] = []
+            $('p').each((_i, elem) => {
+                const paragraphText = $(elem).text()
+                if (paragraphText) {
+                    paragraphs.push(paragraphText.trim())
+                }
+            })
+            const body_text = paragraphs.join(' ').replace(/\s\s+/g, ' ').trim()
+            const foundLinks: string[] = []
+
+            $('a').each((_i, elem) => {
+                const href = $(elem).attr('href')
+                if (href) {
+                    try {
+                        const absoluteUrl = new URL(href, url).toString()
+                        if (absoluteUrl.startsWith('http') && !absoluteUrl.includes('#')) {
+                            foundLinks.push(absoluteUrl)
+                        }
+                    } catch (e) {
+                        // Ignore invalid URLs
+                    }
+                }
+            })
+
+            return {
+                title: title.trim(),
+                description: description.trim(),
+                body_text: body_text,
+                foundLinks: [...new Set(foundLinks)]
+            }
+        } catch (error: any) {
+            if (error.type === 'request-timeout') {
+                return {
+                    title: '',
+                    description: '',
+                    body_text: '',
+                    foundLinks: [],
+                    error: `Scraping Error: Request Timeout after ${this.timeoutMs}ms`
+                }
+            }
+            return {
+                title: '',
+                description: '',
+                body_text: '',
+                foundLinks: [],
+                error: `Scraping Error: ${error?.message || 'Unknown error'}`
+            }
+        }
+    }
+
+    private async scrapeRecursive(url: string, currentDepth: number): Promise<ScrapedPageData[]> {
+        if (this.maxPages !== null && this.scrapedPagesCount >= this.maxPages) {
+            return []
+        }
+        if (currentDepth > this.maxDepth) {
+            return []
+        }
+        if (this.visitedUrls.has(url)) {
+            return []
+        }
+        try {
+            new URL(url)
+            if (!url.startsWith('http')) throw new Error('Invalid protocol')
+        } catch (e) {
+            if (this.maxPages !== null) {
+                this.scrapedPagesCount++
+            }
+            return [{ url, title: '', description: '', body_text: '', error: `Invalid URL format or protocol` }]
+        }
+        this.visitedUrls.add(url)
+        if (this.maxPages !== null) {
+            this.scrapedPagesCount++
+        }
+
+        const { foundLinks, ...scrapedContent } = await this.scrapeSingleUrl(url)
+        const currentPageData: ScrapedPageData = { url, ...scrapedContent }
+        let results: ScrapedPageData[] = [currentPageData]
+
+        if (!currentPageData.error && currentDepth < this.maxDepth && (this.maxPages === null || this.scrapedPagesCount < this.maxPages)) {
+            const recursivePromises: Promise<ScrapedPageData[]>[] = []
+            for (const link of foundLinks) {
+                if (this.maxPages !== null && this.scrapedPagesCount >= this.maxPages) {
+                    break
+                }
+                if (!this.visitedUrls.has(link)) {
+                    recursivePromises.push(this.scrapeRecursive(link, currentDepth + 1))
+                }
+            }
+            if (recursivePromises.length > 0) {
+                const nestedResults = await Promise.all(recursivePromises)
+                results = results.concat(...nestedResults)
+            }
+        } else if (currentPageData.error) {
+            // Do nothing if there was an error scraping the current page
+        }
+        return results
+    }
+
+    private async scrapeUrlsFromList(urlList: string[]): Promise<ScrapedPageData[]> {
+        const results: ScrapedPageData[] = []
+        const scrapePromises: Promise<void>[] = []
+
+        for (const url of urlList) {
+            if (this.maxPages !== null && this.scrapedPagesCount >= this.maxPages) {
+                break
+            }
+            if (this.visitedUrls.has(url)) {
+                continue
+            }
+
+            this.visitedUrls.add(url)
+            this.scrapedPagesCount++
+
+            const promise = (async () => {
+                const { foundLinks: _ignoreLinks, ...scrapedContent } = await this.scrapeSingleUrl(url)
+                results.push({ url, ...scrapedContent })
+            })()
+            scrapePromises.push(promise)
+        }
+
+        await Promise.all(scrapePromises)
+
+        return results.slice(0, this.maxPages ?? results.length)
+    }
+
+    async _call(initialInput: string): Promise<string> {
+        this.visitedUrls = new Set<string>()
+        this.scrapedPagesCount = 0
+        let performedFallback = false
+        let sitemapAttempted = false
+
+        if (!initialInput || typeof initialInput !== 'string') {
+            return JSON.stringify({ error: 'Input must be a single URL string.' })
+        }
+
+        try {
+            let allScrapedData: ScrapedPageData[] = []
+            let urlsFromSitemap: string[] = []
+
+            if (this.useSitemap) {
+                sitemapAttempted = true
+                let sitemapUrlToFetch: string | undefined = undefined
+
+                try {
+                    const baseUrl = new URL(initialInput)
+                    sitemapUrlToFetch = new URL('/sitemap.xml', baseUrl.origin).toString()
+                } catch (e) {
+                    return JSON.stringify({ error: 'Invalid initial URL provided for sitemap detection.' })
+                }
+
+                if (!sitemapUrlToFetch) {
+                    return JSON.stringify({ error: 'Could not determine sitemap URL.' })
+                }
+
+                try {
+                    const limitParam = this.maxPages === null ? Infinity : this.maxPages
+                    urlsFromSitemap = await xmlScrape(sitemapUrlToFetch, limitParam)
+                } catch (sitemapError) {
+                    urlsFromSitemap = []
+                }
+
+                if (urlsFromSitemap.length > 0) {
+                    allScrapedData = await this.scrapeUrlsFromList(urlsFromSitemap)
+                } else {
+                    performedFallback = true
+                }
+            }
+
+            if (!sitemapAttempted || performedFallback) {
+                allScrapedData = await this.scrapeRecursive(initialInput, 1)
+            }
+
+            if (this.maxPages !== null && this.scrapedPagesCount >= this.maxPages) {
+                // Log or indicate that the max page limit was reached during scraping
+            }
+
+            if (performedFallback) {
+                const warningResult = {
+                    warning: 'Sitemap not found or empty; fell back to recursive scraping.',
+                    scrapedData: allScrapedData
+                }
+                return JSON.stringify(warningResult)
+            } else {
+                return JSON.stringify(allScrapedData)
+            }
+        } catch (error: any) {
+            return JSON.stringify({ error: `Failed scrape operation: ${error?.message || 'Unknown error'}` })
+        }
+    }
+}
+
+class WebScraperRecursive_Tools implements INode {
+    label: string
+    name: string
+    version: number
+    description: string
+    type: string
+    icon: string
+    category: string
+    baseClasses: string[]
+    inputs: INodeParams[]
+
+    constructor() {
+        this.label = 'Cheerio Web Scraper'
+        this.name = 'cheerioWebScraper'
+        this.version = 1.0
+        this.type = 'Tool'
+        this.icon = 'cheerioWebScraper.svg'
+        this.category = 'Tools'
+        this.description = 'Scrapes web pages recursively by following links OR by fetching URLs from the default sitemap.'
+        this.baseClasses = [this.type, ...getBaseClasses(WebScraperRecursiveTool)]
+        this.inputs = [
+            {
+                label: 'Scraping Mode',
+                name: 'scrapeMode',
+                type: 'options',
+                options: [
+                    { label: 'Recursive Link Following', name: 'recursive' },
+                    { label: 'Sitemap', name: 'sitemap' }
+                ],
+                default: 'recursive',
+                description:
+                    "Select discovery method: 'Recursive' follows links found on pages (uses Max Depth). 'Sitemap' tries sitemap.xml first, but falls back to 'Recursive' if the sitemap is not found or empty.",
+                additionalParams: true
+            },
+            {
+                label: 'Max Depth',
+                name: 'maxDepth',
+                type: 'number',
+                description:
+                    'Maximum levels of links to follow (e.g., 1 = only the initial URL, 2 = initial URL + links found on it). Default 1.',
+                placeholder: '1',
+                default: 1,
+                optional: true,
+                additionalParams: true
+            },
+            {
+                label: 'Max Pages',
+                name: 'maxPages',
+                type: 'number',
+                description:
+                    'Maximum total number of pages to scrape, regardless of mode or depth. Stops when this limit is reached. Leave empty for no page limit. Default: 10.',
+                placeholder: '10',
+                default: 10,
+                optional: true,
+                additionalParams: true
+            },
+            {
+                label: 'Timeout (s)',
+                name: 'timeoutS',
+                type: 'number',
+                description: 'Maximum time in seconds to wait for each page request to complete. Accepts decimals (e.g., 0.5). Default 60.',
+                placeholder: '60',
+                default: 60,
+                optional: true,
+                additionalParams: true
+            },
+            {
+                label: 'Tool Description',
+                name: 'description',
+                type: 'string',
+                description:
+                    'Custom description of what the tool does. This is for LLM to determine when to use this tool. Overrides the default description.',
+                rows: 4,
+                additionalParams: true,
+                optional: true,
+                placeholder: `Scrapes web pages recursively or via default sitemap. Extracts title, description, and paragraph text. Input should be a single URL string. Returns a JSON string array of scraped page data objects.`
+            }
+        ]
+    }
+
+    async init(nodeData: INodeData, _: string, _options: ICommonObject): Promise<any> {
+        const scrapeMode = (nodeData.inputs?.scrapeMode as string) ?? 'recursive'
+        const useSitemap = scrapeMode === 'sitemap'
+
+        const maxDepthInput = nodeData.inputs?.maxDepth as string | number | undefined
+        let maxDepth = 1
+        if (maxDepthInput !== undefined && maxDepthInput !== '') {
+            const parsedDepth = parseInt(String(maxDepthInput), 10)
+            if (!isNaN(parsedDepth) && parsedDepth > 0) {
+                maxDepth = parsedDepth
+            }
+        }
+
+        const maxPagesInput = nodeData.inputs?.maxPages as string | number | undefined
+        let maxPages: number | null = 10
+        if (maxPagesInput === undefined || maxPagesInput === '') {
+            maxPages = null
+        } else {
+            const parsedPages = parseInt(String(maxPagesInput), 10)
+            if (!isNaN(parsedPages) && parsedPages > 0) {
+                maxPages = parsedPages
+            } else if (parsedPages <= 0) {
+                maxPages = null
+            }
+        }
+
+        const timeoutInputS = nodeData.inputs?.timeoutS as string | number | undefined
+        let timeoutMs = 60000
+        if (timeoutInputS !== undefined && timeoutInputS !== '') {
+            const parsedTimeoutS = parseFloat(String(timeoutInputS))
+            if (!isNaN(parsedTimeoutS) && parsedTimeoutS > 0) {
+                timeoutMs = Math.round(parsedTimeoutS * 1000)
+            }
+        }
+
+        const customDescription = nodeData.inputs?.description as string
+
+        const tool = new WebScraperRecursiveTool(maxDepth, maxPages, timeoutMs, useSitemap)
+
+        if (customDescription) {
+            tool.description = customDescription
+        }
+
+        return tool
+    }
+}
+
+module.exports = { nodeClass: WebScraperRecursive_Tools }

packages/components/nodes/tools/FreeWebScraper/cheeriowebscraper.svg

@@ -0,0 +1 @@
+<svg fill="#000000" height="200px" width="200px" version="1.1" id="Layer_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 0 512 512" xml:space="preserve" stroke="#000000" stroke-width="0.00512"><g id="SVGRepo_bgCarrier" stroke-width="0"></g><g id="SVGRepo_tracerCarrier" stroke-linecap="round" stroke-linejoin="round"></g><g id="SVGRepo_iconCarrier"> <g> <g> <path d="M469.056,297.773h-43.217c-9.425,0-17.067,7.641-17.067,17.067v63.243c0,11.25-9.152,20.403-20.403,20.403 s-20.403-9.152-20.403-20.403V235.55c19.267-27.92,30.092-61.164,30.092-93.49C398.058,63.728,334.331,0,255.999,0 s-142.06,63.728-142.06,142.06c0,32.327,10.825,65.57,30.092,93.49v144.222c0,11.25-9.152,20.403-20.403,20.403 c-11.25,0-20.403-9.152-20.403-20.403v-63.243c0-9.425-7.641-17.067-17.067-17.067H42.942c-9.425,0-17.067,7.641-17.067,17.067 c0,9.425,7.641,17.067,17.067,17.067h26.151v46.177c0,30.071,24.465,54.536,54.536,54.536c30.071,0,54.536-24.464,54.536-54.536 V271.981c8.363,6.505,17.178,11.933,26.332,16.211v116.185c0,40.522-32.968,73.489-73.49,73.489 c-9.425,0-17.067,7.641-17.067,17.067S121.581,512,131.006,512c59.343,0,107.624-48.279,107.624-107.622V298.542 c5.734,0.85,11.533,1.282,17.37,1.282c5.838,0,11.636-0.432,17.37-1.282v105.836c0,59.343,48.28,107.622,107.624,107.622 c9.425,0,17.067-7.641,17.067-17.067s-7.641-17.067-17.067-17.067c-40.522,0-73.49-32.967-73.49-73.489V288.192 c9.152-4.278,17.968-9.706,26.332-16.211v106.103c0,30.071,24.464,54.536,54.536,54.536s54.536-24.465,54.536-54.536v-46.177 h26.151c9.425,0,17.067-7.641,17.067-17.067S478.481,297.773,469.056,297.773z M255.999,265.69 c-59.175,0-107.926-63.631-107.926-123.631c0-59.51,48.415-107.926,107.926-107.926S363.925,82.549,363.925,142.06 C363.925,202.198,315.048,265.69,255.999,265.69z"></path> </g> </g> <g> <g> <circle cx="209.669" cy="204.174" r="22.577"></circle> </g> </g> <g> <g> <circle cx="303.012" cy="204.174" r="22.577"></circle> </g> </g> </g></svg>