diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts index 94a3b538..c7079ed9 100644 --- a/packages/server/src/index.ts +++ b/packages/server/src/index.ts @@ -57,7 +57,7 @@ import { Tool } from './database/entities/Tool' import { Assistant } from './database/entities/Assistant' import { ChatflowPool } from './ChatflowPool' import { CachePool } from './CachePool' -import { ICommonObject, IMessage, INodeOptionsValue, handleEscapeCharacters } from 'flowise-components' +import { ICommonObject, IMessage, INodeOptionsValue, handleEscapeCharacters, webCrawl, xmlScrape } from 'flowise-components' import { createRateLimiter, getRateLimiter, initializeRateLimiter } from './utils/rateLimit' import { addAPIKey, compareKeys, deleteAPIKey, getApiKey, getAPIKeys, updateAPIKey } from './utils/apiKey' import { sanitizeMiddleware } from './utils/XSS' @@ -1087,6 +1087,19 @@ export class App { } }) + // ---------------------------------------- + // Scraper + // ---------------------------------------- + + this.app.get('/api/v1/fetch-links', async (req: Request, res: Response) => { + const url = decodeURIComponent(req.query.url as string) + const relativeLinksMethod = req.query.relativeLinksMethod as string + if (process.env.DEBUG === 'true') console.info(`Start ${relativeLinksMethod}`) + const links: string[] = relativeLinksMethod === 'webCrawl' ? await webCrawl(url, 0) : await xmlScrape(url, 0) + + res.json({ status: 'OK', links }) + }) + // ---------------------------------------- // Upsert // ----------------------------------------