From 92e50a676c27ba29f3168f1720b11b4ba1258b0d Mon Sep 17 00:00:00 2001 From: chungyau97 Date: Wed, 5 Jul 2023 16:47:01 +0800 Subject: [PATCH 01/10] add web crawl --- .../nodes/documentloaders/Cheerio/Cheerio.ts | 48 ++++++----- packages/components/package.json | 1 + packages/components/src/utils.ts | 84 +++++++++++++++++++ 3 files changed, 113 insertions(+), 20 deletions(-) diff --git a/packages/components/nodes/documentloaders/Cheerio/Cheerio.ts b/packages/components/nodes/documentloaders/Cheerio/Cheerio.ts index 9e113505..10eff77e 100644 --- a/packages/components/nodes/documentloaders/Cheerio/Cheerio.ts +++ b/packages/components/nodes/documentloaders/Cheerio/Cheerio.ts @@ -2,7 +2,7 @@ import { INode, INodeData, INodeParams } from '../../../src/Interface' import { TextSplitter } from 'langchain/text_splitter' import { CheerioWebBaseLoader } from 'langchain/document_loaders/web/cheerio' import { test } from 'linkifyjs' -import { getAvailableURLs } from '../../../src' +import { webCrawl } from '../../../src' class Cheerio_DocumentLoaders implements INode { label: string @@ -35,19 +35,20 @@ class Cheerio_DocumentLoaders implements INode { optional: true }, { - label: 'Web Scrap for Relative Links', - name: 'webScrap', + label: 'Web Crawl for Relative Links', + name: 'boolWebCrawl', type: 'boolean', optional: true, additionalParams: true }, { - label: 'Web Scrap Links Limit', + label: 'Web Crawl Links Limit', name: 'limit', type: 'number', default: 10, optional: true, - additionalParams: true + additionalParams: true, + description: 'Set 0 to crawl all relative links' }, { label: 'Metadata', @@ -62,7 +63,7 @@ class Cheerio_DocumentLoaders implements INode { async init(nodeData: INodeData): Promise { const textSplitter = nodeData.inputs?.textSplitter as TextSplitter const metadata = nodeData.inputs?.metadata - const webScrap = nodeData.inputs?.webScrap as boolean + const boolWebCrawl = nodeData.inputs?.boolWebCrawl as boolean let limit = nodeData.inputs?.limit as string let url = nodeData.inputs?.url as string @@ -71,25 +72,32 @@ class Cheerio_DocumentLoaders implements INode { throw new Error('Invalid URL') } - const cheerioLoader = async (url: string): Promise => { - let docs = [] - const loader = new CheerioWebBaseLoader(url) - if (textSplitter) { - docs = await loader.loadAndSplit(textSplitter) - } else { - docs = await loader.load() + async function cheerioLoader(url: string): Promise { + try { + let docs = [] + const loader = new CheerioWebBaseLoader(url) + if (textSplitter) { + docs = await loader.loadAndSplit(textSplitter) + } else { + docs = await loader.load() + } + return docs + } catch (err) { + if (process.env.DEBUG === 'true') console.error(`error in CheerioWebBaseLoader: ${err.message}, on page: ${url}`) } - return docs } - let availableUrls: string[] let docs = [] - if (webScrap) { - if (!limit) limit = '10' - availableUrls = await getAvailableURLs(url, parseInt(limit)) - for (let i = 0; i < availableUrls.length; i++) { - docs.push(...(await cheerioLoader(availableUrls[i]))) + if (boolWebCrawl) { + if (process.env.DEBUG === 'true') console.info('Start Web Crawl') + if (!limit) throw new Error('Please set a limit to crawl') + else if (parseInt(limit) < 0) throw new Error('Limit cannot be less than 0') + const pages: string[] = await webCrawl(url, parseInt(limit)) + if (process.env.DEBUG === 'true') console.info(`pages: ${JSON.stringify(pages)}, length: ${pages.length}`) + for (const page of pages) { + docs.push(...(await cheerioLoader(page))) } + if (process.env.DEBUG === 'true') console.info('Finish Web Crawl') } else { docs = await cheerioLoader(url) } diff --git a/packages/components/package.json b/packages/components/package.json index 4555a883..81f963ed 100644 --- a/packages/components/package.json +++ b/packages/components/package.json @@ -24,6 +24,7 @@ "@qdrant/js-client-rest": "^1.2.2", "@supabase/supabase-js": "^2.21.0", "@types/js-yaml": "^4.0.5", + "@types/jsdom": "^21.1.1", "axios": "^0.27.2", "cheerio": "^1.0.0-rc.12", "chromadb": "^1.4.2", diff --git a/packages/components/src/utils.ts b/packages/components/src/utils.ts index c247ebc2..63bb3969 100644 --- a/packages/components/src/utils.ts +++ b/packages/components/src/utils.ts @@ -2,6 +2,7 @@ import axios from 'axios' import { load } from 'cheerio' import * as fs from 'fs' import * as path from 'path' +import { JSDOM } from 'jsdom' import { BaseCallbackHandler } from 'langchain/callbacks' import { Server } from 'socket.io' import { ChainValues } from 'langchain/dist/schema' @@ -201,6 +202,89 @@ export const getAvailableURLs = async (url: string, limit: number) => { } } +function getURLsFromHTML(htmlBody: string, baseURL: string): string[] { + const dom = new JSDOM(htmlBody) + const linkElements = dom.window.document.querySelectorAll('a') + const urls: string[] = [] + for (const linkElement of linkElements) { + if (linkElement.href.slice(0, 1) === '/') { + try { + const urlObj = new URL(baseURL + linkElement.href) + urls.push(urlObj.href) //relative + } catch (err) { + if (process.env.DEBUG === 'true') console.error(`error with relative url: ${err.message}`) + continue + } + } else { + try { + const urlObj = new URL(linkElement.href) + urls.push(urlObj.href) //absolute + } catch (err) { + if (process.env.DEBUG === 'true') console.error(`error with absolute url: ${err.message}`) + continue + } + } + } + return urls +} + +function normalizeURL(urlString: string): string { + const urlObj = new URL(urlString) + const hostPath = urlObj.hostname + urlObj.pathname + if (hostPath.length > 0 && hostPath.slice(-1) == '/') { + // handling trailing slash + return hostPath.slice(0, -1) + } + return hostPath +} + +export async function crawl(baseURL: string, currentURL: string, pages: string[], limit: number): Promise { + const baseURLObj = new URL(baseURL) + const currentURLObj = new URL(currentURL) + + if (limit !== 0) if (pages.length === limit) return pages + + if (baseURLObj.hostname !== currentURLObj.hostname) return pages + + const normalizeCurrentURL = baseURLObj.protocol + '//' + normalizeURL(currentURL) + if (pages.includes(normalizeCurrentURL)) { + return pages + } + + pages.push(normalizeCurrentURL) + + if (process.env.DEBUG === 'true') console.info(`actively crawling ${currentURL}`) + try { + const resp = await fetch(currentURL) + + if (resp.status > 399) { + if (process.env.DEBUG === 'true') console.error(`error in fetch with status code: ${resp.status}, on page: ${currentURL}`) + return pages + } + + const contentType: string | null = resp.headers.get('content-type') + if ((contentType && !contentType.includes('text/html')) || !contentType) { + if (process.env.DEBUG === 'true') console.error(`non html response, content type: ${contentType}, on page: ${currentURL}`) + return pages + } + + const htmlBody = await resp.text() + const nextURLs = getURLsFromHTML(htmlBody, baseURL) + for (const nextURL of nextURLs) { + pages = await crawl(baseURL, nextURL, pages, limit) + } + } catch (err) { + if (process.env.DEBUG === 'true') console.error(`error in fetch url: ${err.message}, on page: ${currentURL}`) + } + return pages +} + +export async function webCrawl(stringURL: string, limit: number): Promise { + const URLObj = new URL(stringURL) + const modifyURL = stringURL.slice(-1) === '/' ? stringURL.slice(0, -1) : stringURL + return await crawl(URLObj.protocol + '//' + URLObj.hostname, modifyURL, [], limit) +} + /** * Custom chain handler class */ From c18e98761af3dd908df5ae6969289c3dc331bf28 Mon Sep 17 00:00:00 2001 From: chungyau97 Date: Wed, 5 Jul 2023 17:07:45 +0800 Subject: [PATCH 02/10] modify puppeteer web crawl --- .../documentloaders/Puppeteer/Puppeteer.ts | 53 ++++++++++--------- packages/components/src/utils.ts | 2 +- 2 files changed, 29 insertions(+), 26 deletions(-) diff --git a/packages/components/nodes/documentloaders/Puppeteer/Puppeteer.ts b/packages/components/nodes/documentloaders/Puppeteer/Puppeteer.ts index 1331c736..3f27dc03 100644 --- a/packages/components/nodes/documentloaders/Puppeteer/Puppeteer.ts +++ b/packages/components/nodes/documentloaders/Puppeteer/Puppeteer.ts @@ -2,7 +2,7 @@ import { INode, INodeData, INodeParams } from '../../../src/Interface' import { TextSplitter } from 'langchain/text_splitter' import { PuppeteerWebBaseLoader } from 'langchain/document_loaders/web/puppeteer' import { test } from 'linkifyjs' -import { getAvailableURLs } from '../../../src' +import { webCrawl } from '../../../src' class Puppeteer_DocumentLoaders implements INode { label: string @@ -35,19 +35,20 @@ class Puppeteer_DocumentLoaders implements INode { optional: true }, { - label: 'Web Scrape for Relative Links', - name: 'webScrape', + label: 'Web Crawl for Relative Links', + name: 'boolWebCrawl', type: 'boolean', optional: true, additionalParams: true }, { - label: 'Web Scrape Links Limit', + label: 'Web Crawl Links Limit', name: 'limit', type: 'number', default: 10, optional: true, - additionalParams: true + additionalParams: true, + description: 'Set 0 to crawl all relative links' }, { label: 'Metadata', @@ -62,7 +63,7 @@ class Puppeteer_DocumentLoaders implements INode { async init(nodeData: INodeData): Promise { const textSplitter = nodeData.inputs?.textSplitter as TextSplitter const metadata = nodeData.inputs?.metadata - const webScrape = nodeData.inputs?.webScrape as boolean + const boolWebCrawl = nodeData.inputs?.boolWebCrawl as boolean let limit = nodeData.inputs?.limit as string let url = nodeData.inputs?.url as string @@ -71,30 +72,32 @@ class Puppeteer_DocumentLoaders implements INode { throw new Error('Invalid URL') } - const puppeteerLoader = async (url: string): Promise => { - let docs = [] - const loader = new PuppeteerWebBaseLoader(url) - if (textSplitter) { - docs = await loader.loadAndSplit(textSplitter) - } else { - docs = await loader.load() + async function puppeteerLoader(url: string): Promise { + try { + let docs = [] + const loader = new PuppeteerWebBaseLoader(url) + if (textSplitter) { + docs = await loader.loadAndSplit(textSplitter) + } else { + docs = await loader.load() + } + return docs + } catch (err) { + if (process.env.DEBUG === 'true') console.error(`error in CheerioWebBaseLoader: ${err.message}, on page: ${url}`) } - return docs } - let availableUrls: string[] let docs = [] - if (webScrape) { - if (!limit) limit = '10' - availableUrls = await getAvailableURLs(url, parseInt(limit)) - for (let i = 0; i < availableUrls.length; i++) { - try { - docs.push(...(await puppeteerLoader(availableUrls[i]))) - } catch (error) { - console.error('Error loading url with puppeteer. URL: ', availableUrls[i], 'Error: ', error) - continue - } + if (boolWebCrawl) { + if (process.env.DEBUG === 'true') console.info('Start Web Crawl') + if (!limit) throw new Error('Please set a limit to crawl') + else if (parseInt(limit) < 0) throw new Error('Limit cannot be less than 0') + const pages: string[] = await webCrawl(url, parseInt(limit)) + if (process.env.DEBUG === 'true') console.info(`pages: ${JSON.stringify(pages)}, length: ${pages.length}`) + for (const page of pages) { + docs.push(...(await puppeteerLoader(page))) } + if (process.env.DEBUG === 'true') console.info('Finish Web Crawl') } else { docs = await puppeteerLoader(url) } diff --git a/packages/components/src/utils.ts b/packages/components/src/utils.ts index 63bb3969..d99517f0 100644 --- a/packages/components/src/utils.ts +++ b/packages/components/src/utils.ts @@ -242,7 +242,7 @@ export async function crawl(baseURL: string, currentURL: string, pages: string[] const baseURLObj = new URL(baseURL) const currentURLObj = new URL(currentURL) - if (limit !== 0) if (pages.length === limit) return pages + if (limit !== 0 && pages.length === limit) return pages if (baseURLObj.hostname !== currentURLObj.hostname) return pages From 607d4a3394b0a3de05aedef72b4400a6d82ed907 Mon Sep 17 00:00:00 2001 From: chungyau97 Date: Wed, 5 Jul 2023 17:17:13 +0800 Subject: [PATCH 03/10] modify playwright web crawl --- .../documentloaders/Playwright/Playwright.ts | 48 +++++++++++-------- 1 file changed, 28 insertions(+), 20 deletions(-) diff --git a/packages/components/nodes/documentloaders/Playwright/Playwright.ts b/packages/components/nodes/documentloaders/Playwright/Playwright.ts index 6b7790af..6e22d55d 100644 --- a/packages/components/nodes/documentloaders/Playwright/Playwright.ts +++ b/packages/components/nodes/documentloaders/Playwright/Playwright.ts @@ -2,7 +2,7 @@ import { INode, INodeData, INodeParams } from '../../../src/Interface' import { TextSplitter } from 'langchain/text_splitter' import { PlaywrightWebBaseLoader } from 'langchain/document_loaders/web/playwright' import { test } from 'linkifyjs' -import { getAvailableURLs } from '../../../src' +import { webCrawl } from '../../../src' class Playwright_DocumentLoaders implements INode { label: string @@ -35,19 +35,20 @@ class Playwright_DocumentLoaders implements INode { optional: true }, { - label: 'Web Scrap for Relative Links', - name: 'webScrap', + label: 'Web Crawl for Relative Links', + name: 'boolWebCrawl', type: 'boolean', optional: true, additionalParams: true }, { - label: 'Web Scrap Links Limit', + label: 'Web Crawl Links Limit', name: 'limit', type: 'number', default: 10, optional: true, - additionalParams: true + additionalParams: true, + description: 'Set 0 to crawl all relative links' }, { label: 'Metadata', @@ -62,7 +63,7 @@ class Playwright_DocumentLoaders implements INode { async init(nodeData: INodeData): Promise { const textSplitter = nodeData.inputs?.textSplitter as TextSplitter const metadata = nodeData.inputs?.metadata - const webScrap = nodeData.inputs?.webScrap as boolean + const boolWebCrawl = nodeData.inputs?.boolWebCrawl as boolean let limit = nodeData.inputs?.limit as string let url = nodeData.inputs?.url as string @@ -71,25 +72,32 @@ class Playwright_DocumentLoaders implements INode { throw new Error('Invalid URL') } - const playwrightLoader = async (url: string): Promise => { - let docs = [] - const loader = new PlaywrightWebBaseLoader(url) - if (textSplitter) { - docs = await loader.loadAndSplit(textSplitter) - } else { - docs = await loader.load() + async function playwrightLoader(url: string): Promise { + try { + let docs = [] + const loader = new PlaywrightWebBaseLoader(url) + if (textSplitter) { + docs = await loader.loadAndSplit(textSplitter) + } else { + docs = await loader.load() + } + return docs + } catch (err) { + if (process.env.DEBUG === 'true') console.error(`error in CheerioWebBaseLoader: ${err.message}, on page: ${url}`) } - return docs } - let availableUrls: string[] let docs = [] - if (webScrap) { - if (!limit) limit = '10' - availableUrls = await getAvailableURLs(url, parseInt(limit)) - for (let i = 0; i < availableUrls.length; i++) { - docs.push(...(await playwrightLoader(availableUrls[i]))) + if (boolWebCrawl) { + if (process.env.DEBUG === 'true') console.info('Start Web Crawl') + if (!limit) throw new Error('Please set a limit to crawl') + else if (parseInt(limit) < 0) throw new Error('Limit cannot be less than 0') + const pages: string[] = await webCrawl(url, parseInt(limit)) + if (process.env.DEBUG === 'true') console.info(`pages: ${JSON.stringify(pages)}, length: ${pages.length}`) + for (const page of pages) { + docs.push(...(await playwrightLoader(page))) } + if (process.env.DEBUG === 'true') console.info('Finish Web Crawl') } else { docs = await playwrightLoader(url) } From af67d70cfcd0514e3462070a531d539630827d12 Mon Sep 17 00:00:00 2001 From: chungyau97 Date: Wed, 5 Jul 2023 17:28:59 +0800 Subject: [PATCH 04/10] add function desc --- packages/components/src/utils.ts | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/packages/components/src/utils.ts b/packages/components/src/utils.ts index d99517f0..39d7e333 100644 --- a/packages/components/src/utils.ts +++ b/packages/components/src/utils.ts @@ -202,6 +202,9 @@ export const getAvailableURLs = async (url: string, limit: number) => { } } +/** + * Search for href through htmlBody string + */ function getURLsFromHTML(htmlBody: string, baseURL: string): string[] { const dom = new JSDOM(htmlBody) const linkElements = dom.window.document.querySelectorAll('a') @@ -228,6 +231,9 @@ function getURLsFromHTML(htmlBody: string, baseURL: string): string[] { return urls } +/** + * Normalize URL to prevent crawling the same page + */ function normalizeURL(urlString: string): string { const urlObj = new URL(urlString) const hostPath = urlObj.hostname + urlObj.pathname @@ -238,7 +244,10 @@ function normalizeURL(urlString: string): string { return hostPath } -export async function crawl(baseURL: string, currentURL: string, pages: string[], limit: number): Promise { +/** + * Recursive crawl using normalizeURL and getURLsFromHTML + */ +async function crawl(baseURL: string, currentURL: string, pages: string[], limit: number): Promise { const baseURLObj = new URL(baseURL) const currentURLObj = new URL(currentURL) @@ -279,6 +288,9 @@ export async function crawl(baseURL: string, currentURL: string, pages: string[] return pages } +/** + * Prep URL before passing into recursive carwl function + */ export async function webCrawl(stringURL: string, limit: number): Promise { const URLObj = new URL(stringURL) const modifyURL = stringURL.slice(-1) === '/' ? stringURL.slice(0, -1) : stringURL From 6bd44e2d47581849024d4f829bd4e2a361199d69 Mon Sep 17 00:00:00 2001 From: chungyau97 Date: Fri, 7 Jul 2023 00:10:12 +0800 Subject: [PATCH 05/10] cheerio add xml scraper --- .../nodes/documentloaders/Cheerio/Cheerio.ts | 36 +++++++++++------ packages/components/src/utils.ts | 39 +++++++++++++++++++ 2 files changed, 63 insertions(+), 12 deletions(-) diff --git a/packages/components/nodes/documentloaders/Cheerio/Cheerio.ts b/packages/components/nodes/documentloaders/Cheerio/Cheerio.ts index 10eff77e..2106b86f 100644 --- a/packages/components/nodes/documentloaders/Cheerio/Cheerio.ts +++ b/packages/components/nodes/documentloaders/Cheerio/Cheerio.ts @@ -2,7 +2,7 @@ import { INode, INodeData, INodeParams } from '../../../src/Interface' import { TextSplitter } from 'langchain/text_splitter' import { CheerioWebBaseLoader } from 'langchain/document_loaders/web/cheerio' import { test } from 'linkifyjs' -import { webCrawl } from '../../../src' +import { webCrawl, xmlScrape } from '../../../src' class Cheerio_DocumentLoaders implements INode { label: string @@ -35,20 +35,30 @@ class Cheerio_DocumentLoaders implements INode { optional: true }, { - label: 'Web Crawl for Relative Links', - name: 'boolWebCrawl', - type: 'boolean', + label: 'Get Relative Links Method', + name: 'relativeLinksMethod', + type: 'options', + options: [ + { + label: 'Web Crawl', + name: 'webCrawl' + }, + { + label: 'Scrape XML Sitemap', + name: 'scrapeXMLSitemap' + } + ], optional: true, additionalParams: true }, { - label: 'Web Crawl Links Limit', + label: 'Crawl/Scrape Links Limit', name: 'limit', type: 'number', default: 10, optional: true, additionalParams: true, - description: 'Set 0 to crawl all relative links' + description: 'Set 0 to crawl/scrape all relative links' }, { label: 'Metadata', @@ -63,7 +73,7 @@ class Cheerio_DocumentLoaders implements INode { async init(nodeData: INodeData): Promise { const textSplitter = nodeData.inputs?.textSplitter as TextSplitter const metadata = nodeData.inputs?.metadata - const boolWebCrawl = nodeData.inputs?.boolWebCrawl as boolean + const relativeLinksMethod = nodeData.inputs?.relativeLinksMethod as string let limit = nodeData.inputs?.limit as string let url = nodeData.inputs?.url as string @@ -88,16 +98,18 @@ class Cheerio_DocumentLoaders implements INode { } let docs = [] - if (boolWebCrawl) { - if (process.env.DEBUG === 'true') console.info('Start Web Crawl') - if (!limit) throw new Error('Please set a limit to crawl') + if (relativeLinksMethod) { + if (process.env.DEBUG === 'true') console.info(`Start ${relativeLinksMethod}`) + if (!limit) throw new Error('Please set a limit to crawl/scrape') else if (parseInt(limit) < 0) throw new Error('Limit cannot be less than 0') - const pages: string[] = await webCrawl(url, parseInt(limit)) + const pages: string[] = + relativeLinksMethod === 'webCrawl' ? await webCrawl(url, parseInt(limit)) : await xmlScrape(url, parseInt(limit)) if (process.env.DEBUG === 'true') console.info(`pages: ${JSON.stringify(pages)}, length: ${pages.length}`) + if (!pages || pages.length === 0) throw new Error('No relative links found') for (const page of pages) { docs.push(...(await cheerioLoader(page))) } - if (process.env.DEBUG === 'true') console.info('Finish Web Crawl') + if (process.env.DEBUG === 'true') console.info(`Finish ${relativeLinksMethod}`) } else { docs = await cheerioLoader(url) } diff --git a/packages/components/src/utils.ts b/packages/components/src/utils.ts index 39d7e333..a5f69ad9 100644 --- a/packages/components/src/utils.ts +++ b/packages/components/src/utils.ts @@ -297,6 +297,45 @@ export async function webCrawl(stringURL: string, limit: number): Promise { + let urls: string[] = [] + if (process.env.DEBUG === 'true') console.info(`actively scarping ${currentURL}`) + try { + const resp = await fetch(currentURL) + + if (resp.status > 399) { + if (process.env.DEBUG === 'true') console.error(`error in fetch with status code: ${resp.status}, on page: ${currentURL}`) + return urls + } + + const contentType: string | null = resp.headers.get('content-type') + if ((contentType && !contentType.includes('application/xml')) || !contentType) { + if (process.env.DEBUG === 'true') console.error(`non xml response, content type: ${contentType}, on page: ${currentURL}`) + return urls + } + + const xmlBody = await resp.text() + urls = getURLsFromXML(xmlBody, limit) + } catch (err) { + if (process.env.DEBUG === 'true') console.error(`error in fetch url: ${err.message}, on page: ${currentURL}`) + } + return urls +} + /** * Custom chain handler class */ From c2523d8eecc75e434d51d07abb1317bbcae4740a Mon Sep 17 00:00:00 2001 From: chungyau97 Date: Fri, 7 Jul 2023 00:21:11 +0800 Subject: [PATCH 06/10] puppeteer add xml scraper --- .../documentloaders/Puppeteer/Puppeteer.ts | 36 ++++++++++++------- 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/packages/components/nodes/documentloaders/Puppeteer/Puppeteer.ts b/packages/components/nodes/documentloaders/Puppeteer/Puppeteer.ts index 3f27dc03..101a41ea 100644 --- a/packages/components/nodes/documentloaders/Puppeteer/Puppeteer.ts +++ b/packages/components/nodes/documentloaders/Puppeteer/Puppeteer.ts @@ -2,7 +2,7 @@ import { INode, INodeData, INodeParams } from '../../../src/Interface' import { TextSplitter } from 'langchain/text_splitter' import { PuppeteerWebBaseLoader } from 'langchain/document_loaders/web/puppeteer' import { test } from 'linkifyjs' -import { webCrawl } from '../../../src' +import { webCrawl, xmlScrape } from '../../../src' class Puppeteer_DocumentLoaders implements INode { label: string @@ -35,20 +35,30 @@ class Puppeteer_DocumentLoaders implements INode { optional: true }, { - label: 'Web Crawl for Relative Links', - name: 'boolWebCrawl', - type: 'boolean', + label: 'Get Relative Links Method', + name: 'relativeLinksMethod', + type: 'options', + options: [ + { + label: 'Web Crawl', + name: 'webCrawl' + }, + { + label: 'Scrape XML Sitemap', + name: 'scrapeXMLSitemap' + } + ], optional: true, additionalParams: true }, { - label: 'Web Crawl Links Limit', + label: 'Crawl/Scrape Links Limit', name: 'limit', type: 'number', default: 10, optional: true, additionalParams: true, - description: 'Set 0 to crawl all relative links' + description: 'Set 0 to crawl/scrape all relative links' }, { label: 'Metadata', @@ -63,7 +73,7 @@ class Puppeteer_DocumentLoaders implements INode { async init(nodeData: INodeData): Promise { const textSplitter = nodeData.inputs?.textSplitter as TextSplitter const metadata = nodeData.inputs?.metadata - const boolWebCrawl = nodeData.inputs?.boolWebCrawl as boolean + const relativeLinksMethod = nodeData.inputs?.relativeLinksMethod as string let limit = nodeData.inputs?.limit as string let url = nodeData.inputs?.url as string @@ -88,16 +98,18 @@ class Puppeteer_DocumentLoaders implements INode { } let docs = [] - if (boolWebCrawl) { - if (process.env.DEBUG === 'true') console.info('Start Web Crawl') - if (!limit) throw new Error('Please set a limit to crawl') + if (relativeLinksMethod) { + if (process.env.DEBUG === 'true') console.info(`Start ${relativeLinksMethod}`) + if (!limit) throw new Error('Please set a limit to crawl/scrape') else if (parseInt(limit) < 0) throw new Error('Limit cannot be less than 0') - const pages: string[] = await webCrawl(url, parseInt(limit)) + const pages: string[] = + relativeLinksMethod === 'webCrawl' ? await webCrawl(url, parseInt(limit)) : await xmlScrape(url, parseInt(limit)) if (process.env.DEBUG === 'true') console.info(`pages: ${JSON.stringify(pages)}, length: ${pages.length}`) + if (!pages || pages.length === 0) throw new Error('No relative links found') for (const page of pages) { docs.push(...(await puppeteerLoader(page))) } - if (process.env.DEBUG === 'true') console.info('Finish Web Crawl') + if (process.env.DEBUG === 'true') console.info(`Finish ${relativeLinksMethod}`) } else { docs = await puppeteerLoader(url) } From e4a488c211b05c5e4f6d126fb4cabb176f53dafd Mon Sep 17 00:00:00 2001 From: chungyau97 Date: Fri, 7 Jul 2023 00:37:10 +0800 Subject: [PATCH 07/10] playwright add xml scraper --- .../documentloaders/Playwright/Playwright.ts | 36 ++++++++++++------- 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/packages/components/nodes/documentloaders/Playwright/Playwright.ts b/packages/components/nodes/documentloaders/Playwright/Playwright.ts index 6e22d55d..c02ab442 100644 --- a/packages/components/nodes/documentloaders/Playwright/Playwright.ts +++ b/packages/components/nodes/documentloaders/Playwright/Playwright.ts @@ -2,7 +2,7 @@ import { INode, INodeData, INodeParams } from '../../../src/Interface' import { TextSplitter } from 'langchain/text_splitter' import { PlaywrightWebBaseLoader } from 'langchain/document_loaders/web/playwright' import { test } from 'linkifyjs' -import { webCrawl } from '../../../src' +import { webCrawl, xmlScrape } from '../../../src' class Playwright_DocumentLoaders implements INode { label: string @@ -35,20 +35,30 @@ class Playwright_DocumentLoaders implements INode { optional: true }, { - label: 'Web Crawl for Relative Links', - name: 'boolWebCrawl', - type: 'boolean', + label: 'Get Relative Links Method', + name: 'relativeLinksMethod', + type: 'options', + options: [ + { + label: 'Web Crawl', + name: 'webCrawl' + }, + { + label: 'Scrape XML Sitemap', + name: 'scrapeXMLSitemap' + } + ], optional: true, additionalParams: true }, { - label: 'Web Crawl Links Limit', + label: 'Crawl/Scrape Links Limit', name: 'limit', type: 'number', default: 10, optional: true, additionalParams: true, - description: 'Set 0 to crawl all relative links' + description: 'Set 0 to crawl/scrape all relative links' }, { label: 'Metadata', @@ -63,7 +73,7 @@ class Playwright_DocumentLoaders implements INode { async init(nodeData: INodeData): Promise { const textSplitter = nodeData.inputs?.textSplitter as TextSplitter const metadata = nodeData.inputs?.metadata - const boolWebCrawl = nodeData.inputs?.boolWebCrawl as boolean + const relativeLinksMethod = nodeData.inputs?.relativeLinksMethod as string let limit = nodeData.inputs?.limit as string let url = nodeData.inputs?.url as string @@ -88,16 +98,18 @@ class Playwright_DocumentLoaders implements INode { } let docs = [] - if (boolWebCrawl) { - if (process.env.DEBUG === 'true') console.info('Start Web Crawl') - if (!limit) throw new Error('Please set a limit to crawl') + if (relativeLinksMethod) { + if (process.env.DEBUG === 'true') console.info(`Start ${relativeLinksMethod}`) + if (!limit) throw new Error('Please set a limit to crawl/scrape') else if (parseInt(limit) < 0) throw new Error('Limit cannot be less than 0') - const pages: string[] = await webCrawl(url, parseInt(limit)) + const pages: string[] = + relativeLinksMethod === 'webCrawl' ? await webCrawl(url, parseInt(limit)) : await xmlScrape(url, parseInt(limit)) if (process.env.DEBUG === 'true') console.info(`pages: ${JSON.stringify(pages)}, length: ${pages.length}`) + if (!pages || pages.length === 0) throw new Error('No relative links found') for (const page of pages) { docs.push(...(await playwrightLoader(page))) } - if (process.env.DEBUG === 'true') console.info('Finish Web Crawl') + if (process.env.DEBUG === 'true') console.info(`Finish ${relativeLinksMethod}`) } else { docs = await playwrightLoader(url) } From 7e076028dfdb3fe728648fbadab95dd6b3df59f7 Mon Sep 17 00:00:00 2001 From: Henry Date: Sat, 8 Jul 2023 01:55:44 +0100 Subject: [PATCH 08/10] add warning description --- .../nodes/documentloaders/Cheerio/Cheerio.ts | 3 ++- .../documentloaders/Playwright/Playwright.ts | 3 ++- .../documentloaders/Puppeteer/Puppeteer.ts | 3 ++- packages/components/src/Interface.ts | 1 + .../ui/src/views/canvas/NodeInputHandler.js | 18 +++++++++++++++++- 5 files changed, 24 insertions(+), 4 deletions(-) diff --git a/packages/components/nodes/documentloaders/Cheerio/Cheerio.ts b/packages/components/nodes/documentloaders/Cheerio/Cheerio.ts index 2106b86f..2521b039 100644 --- a/packages/components/nodes/documentloaders/Cheerio/Cheerio.ts +++ b/packages/components/nodes/documentloaders/Cheerio/Cheerio.ts @@ -58,7 +58,8 @@ class Cheerio_DocumentLoaders implements INode { default: 10, optional: true, additionalParams: true, - description: 'Set 0 to crawl/scrape all relative links' + description: 'Set 0 to crawl/scrape all relative links', + warning: `Scraping all links might take long time, and all links will be upserted again if the flow's state changed (eg: different URL, chunk size, etc) ` }, { label: 'Metadata', diff --git a/packages/components/nodes/documentloaders/Playwright/Playwright.ts b/packages/components/nodes/documentloaders/Playwright/Playwright.ts index c02ab442..2301b4e9 100644 --- a/packages/components/nodes/documentloaders/Playwright/Playwright.ts +++ b/packages/components/nodes/documentloaders/Playwright/Playwright.ts @@ -58,7 +58,8 @@ class Playwright_DocumentLoaders implements INode { default: 10, optional: true, additionalParams: true, - description: 'Set 0 to crawl/scrape all relative links' + description: 'Set 0 to crawl/scrape all relative links', + warning: `Scraping all links might take long time, and all links will be upserted again if the flow's state changed (eg: different URL, chunk size, etc) ` }, { label: 'Metadata', diff --git a/packages/components/nodes/documentloaders/Puppeteer/Puppeteer.ts b/packages/components/nodes/documentloaders/Puppeteer/Puppeteer.ts index 101a41ea..bf0920bb 100644 --- a/packages/components/nodes/documentloaders/Puppeteer/Puppeteer.ts +++ b/packages/components/nodes/documentloaders/Puppeteer/Puppeteer.ts @@ -58,7 +58,8 @@ class Puppeteer_DocumentLoaders implements INode { default: 10, optional: true, additionalParams: true, - description: 'Set 0 to crawl/scrape all relative links' + description: 'Set 0 to crawl/scrape all relative links', + warning: `Scraping all links might take long time, and all links will be upserted again if the flow's state changed (eg: different URL, chunk size, etc) ` }, { label: 'Metadata', diff --git a/packages/components/src/Interface.ts b/packages/components/src/Interface.ts index f8a6fd58..d9233e49 100644 --- a/packages/components/src/Interface.ts +++ b/packages/components/src/Interface.ts @@ -57,6 +57,7 @@ export interface INodeParams { type: NodeParamsType | string default?: CommonType | ICommonObject | ICommonObject[] description?: string + warning?: string options?: Array optional?: boolean | INodeDisplay rows?: number diff --git a/packages/ui/src/views/canvas/NodeInputHandler.js b/packages/ui/src/views/canvas/NodeInputHandler.js index 2d96bcb5..ba72a4ce 100644 --- a/packages/ui/src/views/canvas/NodeInputHandler.js +++ b/packages/ui/src/views/canvas/NodeInputHandler.js @@ -7,7 +7,7 @@ import { useSelector } from 'react-redux' import { useTheme, styled } from '@mui/material/styles' import { Box, Typography, Tooltip, IconButton, Button } from '@mui/material' import { tooltipClasses } from '@mui/material/Tooltip' -import { IconArrowsMaximize, IconEdit } from '@tabler/icons' +import { IconArrowsMaximize, IconEdit, IconAlertTriangle } from '@tabler/icons' // project import import { Dropdown } from 'ui-component/dropdown/Dropdown' @@ -210,6 +210,22 @@ const NodeInputHandler = ({ inputAnchor, inputParam, data, disabled = false, isA )} + {inputParam.warning && ( +
+ + {inputParam.warning} +
+ )} {inputParam.type === 'file' && ( Date: Sat, 8 Jul 2023 11:36:31 +0100 Subject: [PATCH 09/10] update error message --- .../components/nodes/documentloaders/Playwright/Playwright.ts | 2 +- .../components/nodes/documentloaders/Puppeteer/Puppeteer.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/components/nodes/documentloaders/Playwright/Playwright.ts b/packages/components/nodes/documentloaders/Playwright/Playwright.ts index 2301b4e9..273536ae 100644 --- a/packages/components/nodes/documentloaders/Playwright/Playwright.ts +++ b/packages/components/nodes/documentloaders/Playwright/Playwright.ts @@ -94,7 +94,7 @@ class Playwright_DocumentLoaders implements INode { } return docs } catch (err) { - if (process.env.DEBUG === 'true') console.error(`error in CheerioWebBaseLoader: ${err.message}, on page: ${url}`) + if (process.env.DEBUG === 'true') console.error(`error in PlaywrightWebBaseLoader: ${err.message}, on page: ${url}`) } } diff --git a/packages/components/nodes/documentloaders/Puppeteer/Puppeteer.ts b/packages/components/nodes/documentloaders/Puppeteer/Puppeteer.ts index bf0920bb..86a9477d 100644 --- a/packages/components/nodes/documentloaders/Puppeteer/Puppeteer.ts +++ b/packages/components/nodes/documentloaders/Puppeteer/Puppeteer.ts @@ -94,7 +94,7 @@ class Puppeteer_DocumentLoaders implements INode { } return docs } catch (err) { - if (process.env.DEBUG === 'true') console.error(`error in CheerioWebBaseLoader: ${err.message}, on page: ${url}`) + if (process.env.DEBUG === 'true') console.error(`error in PuppeteerWebBaseLoader: ${err.message}, on page: ${url}`) } } From d53522a0a800c11f209379210b1a27b9569dcd59 Mon Sep 17 00:00:00 2001 From: chungyau97 Date: Sun, 9 Jul 2023 10:32:05 +0800 Subject: [PATCH 10/10] add description and modify default limit to 10 if empty --- .../nodes/documentloaders/Cheerio/Cheerio.ts | 17 ++++++++++------- .../documentloaders/Playwright/Playwright.ts | 17 ++++++++++------- .../documentloaders/Puppeteer/Puppeteer.ts | 17 ++++++++++------- 3 files changed, 30 insertions(+), 21 deletions(-) diff --git a/packages/components/nodes/documentloaders/Cheerio/Cheerio.ts b/packages/components/nodes/documentloaders/Cheerio/Cheerio.ts index 2521b039..b93a8685 100644 --- a/packages/components/nodes/documentloaders/Cheerio/Cheerio.ts +++ b/packages/components/nodes/documentloaders/Cheerio/Cheerio.ts @@ -38,28 +38,31 @@ class Cheerio_DocumentLoaders implements INode { label: 'Get Relative Links Method', name: 'relativeLinksMethod', type: 'options', + description: 'Select a method to retrieve relative links', options: [ { label: 'Web Crawl', - name: 'webCrawl' + name: 'webCrawl', + description: 'Crawl relative links from HTML URL' }, { label: 'Scrape XML Sitemap', - name: 'scrapeXMLSitemap' + name: 'scrapeXMLSitemap', + description: 'Scrape relative links from XML sitemap URL' } ], optional: true, additionalParams: true }, { - label: 'Crawl/Scrape Links Limit', + label: 'Get Relative Links Limit', name: 'limit', type: 'number', - default: 10, optional: true, additionalParams: true, - description: 'Set 0 to crawl/scrape all relative links', - warning: `Scraping all links might take long time, and all links will be upserted again if the flow's state changed (eg: different URL, chunk size, etc) ` + description: + 'Only used when "Get Relative Links Method" is selected. Set 0 to retrieve all relative links, default limit is 10.', + warning: `Retreiving all links might take long time, and all links will be upserted again if the flow's state changed (eg: different URL, chunk size, etc)` }, { label: 'Metadata', @@ -101,7 +104,7 @@ class Cheerio_DocumentLoaders implements INode { let docs = [] if (relativeLinksMethod) { if (process.env.DEBUG === 'true') console.info(`Start ${relativeLinksMethod}`) - if (!limit) throw new Error('Please set a limit to crawl/scrape') + if (!limit) limit = '10' else if (parseInt(limit) < 0) throw new Error('Limit cannot be less than 0') const pages: string[] = relativeLinksMethod === 'webCrawl' ? await webCrawl(url, parseInt(limit)) : await xmlScrape(url, parseInt(limit)) diff --git a/packages/components/nodes/documentloaders/Playwright/Playwright.ts b/packages/components/nodes/documentloaders/Playwright/Playwright.ts index 273536ae..73a3e290 100644 --- a/packages/components/nodes/documentloaders/Playwright/Playwright.ts +++ b/packages/components/nodes/documentloaders/Playwright/Playwright.ts @@ -38,28 +38,31 @@ class Playwright_DocumentLoaders implements INode { label: 'Get Relative Links Method', name: 'relativeLinksMethod', type: 'options', + description: 'Select a method to retrieve relative links', options: [ { label: 'Web Crawl', - name: 'webCrawl' + name: 'webCrawl', + description: 'Crawl relative links from HTML URL' }, { label: 'Scrape XML Sitemap', - name: 'scrapeXMLSitemap' + name: 'scrapeXMLSitemap', + description: 'Scrape relative links from XML sitemap URL' } ], optional: true, additionalParams: true }, { - label: 'Crawl/Scrape Links Limit', + label: 'Get Relative Links Limit', name: 'limit', type: 'number', - default: 10, optional: true, additionalParams: true, - description: 'Set 0 to crawl/scrape all relative links', - warning: `Scraping all links might take long time, and all links will be upserted again if the flow's state changed (eg: different URL, chunk size, etc) ` + description: + 'Only used when "Get Relative Links Method" is selected. Set 0 to retrieve all relative links, default limit is 10.', + warning: `Retreiving all links might take long time, and all links will be upserted again if the flow's state changed (eg: different URL, chunk size, etc)` }, { label: 'Metadata', @@ -101,7 +104,7 @@ class Playwright_DocumentLoaders implements INode { let docs = [] if (relativeLinksMethod) { if (process.env.DEBUG === 'true') console.info(`Start ${relativeLinksMethod}`) - if (!limit) throw new Error('Please set a limit to crawl/scrape') + if (!limit) limit = '10' else if (parseInt(limit) < 0) throw new Error('Limit cannot be less than 0') const pages: string[] = relativeLinksMethod === 'webCrawl' ? await webCrawl(url, parseInt(limit)) : await xmlScrape(url, parseInt(limit)) diff --git a/packages/components/nodes/documentloaders/Puppeteer/Puppeteer.ts b/packages/components/nodes/documentloaders/Puppeteer/Puppeteer.ts index 618d110b..014845d2 100644 --- a/packages/components/nodes/documentloaders/Puppeteer/Puppeteer.ts +++ b/packages/components/nodes/documentloaders/Puppeteer/Puppeteer.ts @@ -38,28 +38,31 @@ class Puppeteer_DocumentLoaders implements INode { label: 'Get Relative Links Method', name: 'relativeLinksMethod', type: 'options', + description: 'Select a method to retrieve relative links', options: [ { label: 'Web Crawl', - name: 'webCrawl' + name: 'webCrawl', + description: 'Crawl relative links from HTML URL' }, { label: 'Scrape XML Sitemap', - name: 'scrapeXMLSitemap' + name: 'scrapeXMLSitemap', + description: 'Scrape relative links from XML sitemap URL' } ], optional: true, additionalParams: true }, { - label: 'Crawl/Scrape Links Limit', + label: 'Get Relative Links Limit', name: 'limit', type: 'number', - default: 10, optional: true, additionalParams: true, - description: 'Set 0 to crawl/scrape all relative links', - warning: `Scraping all links might take long time, and all links will be upserted again if the flow's state changed (eg: different URL, chunk size, etc) ` + description: + 'Only used when "Get Relative Links Method" is selected. Set 0 to retrieve all relative links, default limit is 10.', + warning: `Retreiving all links might take long time, and all links will be upserted again if the flow's state changed (eg: different URL, chunk size, etc)` }, { label: 'Metadata', @@ -106,7 +109,7 @@ class Puppeteer_DocumentLoaders implements INode { let docs = [] if (relativeLinksMethod) { if (process.env.DEBUG === 'true') console.info(`Start ${relativeLinksMethod}`) - if (!limit) throw new Error('Please set a limit to crawl/scrape') + if (!limit) limit = '10' else if (parseInt(limit) < 0) throw new Error('Limit cannot be less than 0') const pages: string[] = relativeLinksMethod === 'webCrawl' ? await webCrawl(url, parseInt(limit)) : await xmlScrape(url, parseInt(limit))