add web crawl

2026-06-28 13:00:56 +03:00 · 2023-07-05 16:47:01 +08:00
parent 7608d74676
commit 92e50a676c
3 changed files with 113 additions and 20 deletions
@@ -2,7 +2,7 @@ import { INode, INodeData, INodeParams } from '../../../src/Interface'
 import { TextSplitter } from 'langchain/text_splitter'
 import { CheerioWebBaseLoader } from 'langchain/document_loaders/web/cheerio'
 import { test } from 'linkifyjs'
-import { getAvailableURLs } from '../../../src'
+import { webCrawl } from '../../../src'

 class Cheerio_DocumentLoaders implements INode {
    label: string
@@ -35,19 +35,20 @@ class Cheerio_DocumentLoaders implements INode {
                optional: true
            },
            {
-                label: 'Web Scrap for Relative Links',
-                name: 'webScrap',
+                label: 'Web Crawl for Relative Links',
+                name: 'boolWebCrawl',
                type: 'boolean',
                optional: true,
                additionalParams: true
            },
            {
-                label: 'Web Scrap Links Limit',
+                label: 'Web Crawl Links Limit',
                name: 'limit',
                type: 'number',
                default: 10,
                optional: true,
-                additionalParams: true
+                additionalParams: true,
+                description: 'Set 0 to crawl all relative links'
            },
            {
                label: 'Metadata',
@@ -62,7 +63,7 @@ class Cheerio_DocumentLoaders implements INode {
    async init(nodeData: INodeData): Promise<any> {
        const textSplitter = nodeData.inputs?.textSplitter as TextSplitter
        const metadata = nodeData.inputs?.metadata
-        const webScrap = nodeData.inputs?.webScrap as boolean
+        const boolWebCrawl = nodeData.inputs?.boolWebCrawl as boolean
        let limit = nodeData.inputs?.limit as string

        let url = nodeData.inputs?.url as string
@@ -71,25 +72,32 @@ class Cheerio_DocumentLoaders implements INode {
            throw new Error('Invalid URL')
        }

-        const cheerioLoader = async (url: string): Promise<any> => {
-            let docs = []
-            const loader = new CheerioWebBaseLoader(url)
-            if (textSplitter) {
-                docs = await loader.loadAndSplit(textSplitter)
-            } else {
-                docs = await loader.load()
+        async function cheerioLoader(url: string): Promise<any> {
+            try {
+                let docs = []
+                const loader = new CheerioWebBaseLoader(url)
+                if (textSplitter) {
+                    docs = await loader.loadAndSplit(textSplitter)
+                } else {
+                    docs = await loader.load()
+                }
+                return docs
+            } catch (err) {
+                if (process.env.DEBUG === 'true') console.error(`error in CheerioWebBaseLoader: ${err.message}, on page: ${url}`)
            }
-            return docs
        }

-        let availableUrls: string[]
        let docs = []
-        if (webScrap) {
-            if (!limit) limit = '10'
-            availableUrls = await getAvailableURLs(url, parseInt(limit))
-            for (let i = 0; i < availableUrls.length; i++) {
-                docs.push(...(await cheerioLoader(availableUrls[i])))
+        if (boolWebCrawl) {
+            if (process.env.DEBUG === 'true') console.info('Start Web Crawl')
+            if (!limit) throw new Error('Please set a limit to crawl')
+            else if (parseInt(limit) < 0) throw new Error('Limit cannot be less than 0')
+            const pages: string[] = await webCrawl(url, parseInt(limit))
+            if (process.env.DEBUG === 'true') console.info(`pages: ${JSON.stringify(pages)}, length: ${pages.length}`)
+            for (const page of pages) {
+                docs.push(...(await cheerioLoader(page)))
            }
+            if (process.env.DEBUG === 'true') console.info('Finish Web Crawl')
        } else {
            docs = await cheerioLoader(url)
        }
@@ -24,6 +24,7 @@
        "@qdrant/js-client-rest": "^1.2.2",
        "@supabase/supabase-js": "^2.21.0",
        "@types/js-yaml": "^4.0.5",
+        "@types/jsdom": "^21.1.1",
        "axios": "^0.27.2",
        "cheerio": "^1.0.0-rc.12",
        "chromadb": "^1.4.2",
@@ -2,6 +2,7 @@ import axios from 'axios'
 import { load } from 'cheerio'
 import * as fs from 'fs'
 import * as path from 'path'
+import { JSDOM } from 'jsdom'
 import { BaseCallbackHandler } from 'langchain/callbacks'
 import { Server } from 'socket.io'
 import { ChainValues } from 'langchain/dist/schema'
@@ -201,6 +202,89 @@ export const getAvailableURLs = async (url: string, limit: number) => {
    }
 }

+function getURLsFromHTML(htmlBody: string, baseURL: string): string[] {
+    const dom = new JSDOM(htmlBody)
+    const linkElements = dom.window.document.querySelectorAll('a')
+    const urls: string[] = []
+    for (const linkElement of linkElements) {
+        if (linkElement.href.slice(0, 1) === '/') {
+            try {
+                const urlObj = new URL(baseURL + linkElement.href)
+                urls.push(urlObj.href) //relative
+            } catch (err) {
+                if (process.env.DEBUG === 'true') console.error(`error with relative url: ${err.message}`)
+                continue
+            }
+        } else {
+            try {
+                const urlObj = new URL(linkElement.href)
+                urls.push(urlObj.href) //absolute
+            } catch (err) {
+                if (process.env.DEBUG === 'true') console.error(`error with absolute url: ${err.message}`)
+                continue
+            }
+        }
+    }
+    return urls
+}
+
+function normalizeURL(urlString: string): string {
+    const urlObj = new URL(urlString)
+    const hostPath = urlObj.hostname + urlObj.pathname
+    if (hostPath.length > 0 && hostPath.slice(-1) == '/') {
+        // handling trailing slash
+        return hostPath.slice(0, -1)
+    }
+    return hostPath
+}
+
+export async function crawl(baseURL: string, currentURL: string, pages: string[], limit: number): Promise<string[]> {
+    const baseURLObj = new URL(baseURL)
+    const currentURLObj = new URL(currentURL)
+
+    if (limit !== 0) if (pages.length === limit) return pages
+
+    if (baseURLObj.hostname !== currentURLObj.hostname) return pages
+
+    const normalizeCurrentURL = baseURLObj.protocol + '//' + normalizeURL(currentURL)
+    if (pages.includes(normalizeCurrentURL)) {
+        return pages
+    }
+
+    pages.push(normalizeCurrentURL)
+
+    if (process.env.DEBUG === 'true') console.info(`actively crawling ${currentURL}`)
+    try {
+        const resp = await fetch(currentURL)
+
+        if (resp.status > 399) {
+            if (process.env.DEBUG === 'true') console.error(`error in fetch with status code: ${resp.status}, on page: ${currentURL}`)
+            return pages
+        }
+
+        const contentType: string | null = resp.headers.get('content-type')
+        if ((contentType && !contentType.includes('text/html')) || !contentType) {
+            if (process.env.DEBUG === 'true') console.error(`non html response, content type: ${contentType}, on page: ${currentURL}`)
+            return pages
+        }
+
+        const htmlBody = await resp.text()
+        const nextURLs = getURLsFromHTML(htmlBody, baseURL)
+        for (const nextURL of nextURLs) {
+            pages = await crawl(baseURL, nextURL, pages, limit)
+        }
+    } catch (err) {
+        if (process.env.DEBUG === 'true') console.error(`error in fetch url: ${err.message}, on page: ${currentURL}`)
+    }
+    return pages
+}
+
+export async function webCrawl(stringURL: string, limit: number): Promise<string[]> {
+    const URLObj = new URL(stringURL)
+    const modifyURL = stringURL.slice(-1) === '/' ? stringURL.slice(0, -1) : stringURL
+    return await crawl(URLObj.protocol + '//' + URLObj.hostname, modifyURL, [], limit)
+}
+
 /**
 * Custom chain handler class
 */