add web scrap for relative links

packages/components/nodes/documentloaders/Cheerio/Cheerio.ts

@@ -2,6 +2,7 @@ import { INode, INodeData, INodeParams } from '../../../src/Interface'
 import { TextSplitter } from 'langchain/text_splitter'
 import { CheerioWebBaseLoader } from 'langchain/document_loaders/web/cheerio'
 import { test } from 'linkifyjs'
+import { getAvailableURLs } from '../../../src'
 class Cheerio_DocumentLoaders implements INode {
     label: string
     name: string
@@ -38,6 +39,21 @@ class Cheerio_DocumentLoaders implements INode {
                 type: 'json',
                 optional: true,
                 additionalParams: true
+            },
+            {
+                label: 'Web Scrap for Relative Links',
+                name: 'webScrap',
+                type: 'boolean',
+                optional: true,
+                additionalParams: true
+            },
+            {
+                label: 'Web Scrap Links Limit',
+                name: 'limit',
+                type: 'number',
+                default: 10,
+                optional: true,
+                additionalParams: true
             }
         ]
     }
@@ -45,6 +61,8 @@ class Cheerio_DocumentLoaders implements INode {
     async init(nodeData: INodeData): Promise<any> {
         const textSplitter = nodeData.inputs?.textSplitter as TextSplitter
         const metadata = nodeData.inputs?.metadata
+        const webScrap = nodeData.inputs?.webScrap as boolean
+        let limit = nodeData.inputs?.limit as string
 
         let url = nodeData.inputs?.url as string
         url = url.trim()
@@ -52,13 +70,27 @@ class Cheerio_DocumentLoaders implements INode {
             throw new Error('Invalid URL')
         }
 
-        const loader = new CheerioWebBaseLoader(url)
-        let docs = []
+        const cheerioLoader = async (url: string): Promise<any> => {
+            let docs = []
+            const loader = new CheerioWebBaseLoader(url)
+            if (textSplitter) {
+                docs = await loader.loadAndSplit(textSplitter)
+            } else {
+                docs = await loader.load()
+            }
+            return docs
+        }
 
-        if (textSplitter) {
-            docs = await loader.loadAndSplit(textSplitter)
+        let availableUrls: string[]
+        let docs = []
+        if (webScrap) {
+            if (!limit) limit = "10"
+            availableUrls = await getAvailableURLs(url, parseInt(limit))
+            for (let i = 0; i < availableUrls.length; i++) {
+                docs.push(...(await cheerioLoader(availableUrls[i])))
+            }
         } else {
-            docs = await loader.load()
+            docs = await cheerioLoader(url)
         }
 
         if (metadata) {