Merge pull request #1687 from 0xi4o/bug/scrap-limit

Fix: relative links method and limit not applying to manage links

packages/ui/src/api/scraper.js

@@ -1,8 +1,8 @@
 import client from './client'
 
-const fetchAllLinks = (url, relativeLinksMethod) =>
-    client.get(`/fetch-links?url=${encodeURIComponent(url)}&relativeLinksMethod=${relativeLinksMethod}`)
+const fetchLinks = (url, relativeLinksMethod, relativeLinksLimit) =>
+    client.get(`/fetch-links?url=${encodeURIComponent(url)}&relativeLinksMethod=${relativeLinksMethod}&limit=${relativeLinksLimit}`)
 
 export default {
-    fetchAllLinks
+    fetchLinks
 }

packages/ui/src/ui-component/dialog/ManageScrapedLinksDialog.js

@@ -16,7 +16,7 @@ import {
     Stack,
     Typography
 } from '@mui/material'
-import { IconTrash } from '@tabler/icons'
+import { IconTrash, IconX } from '@tabler/icons'
 import PerfectScrollbar from 'react-perfect-scrollbar'
 
 import { BackdropLoader } from 'ui-component/loading/BackdropLoader'
@@ -24,12 +24,23 @@ import { StyledButton } from 'ui-component/button/StyledButton'
 
 import scraperApi from 'api/scraper'
 
-import { HIDE_CANVAS_DIALOG, SHOW_CANVAS_DIALOG } from 'store/actions'
+import useNotifier from 'utils/useNotifier'
+
+import {
+    HIDE_CANVAS_DIALOG,
+    SHOW_CANVAS_DIALOG,
+    enqueueSnackbar as enqueueSnackbarAction,
+    closeSnackbar as closeSnackbarAction
+} from 'store/actions'
 
 const ManageScrapedLinksDialog = ({ show, dialogProps, onCancel, onSave }) => {
     const portalElement = document.getElementById('portal')
     const dispatch = useDispatch()
 
+    useNotifier()
+    const enqueueSnackbar = (...args) => dispatch(enqueueSnackbarAction(...args))
+    const closeSnackbar = (...args) => dispatch(closeSnackbarAction(...args))
+
     const [loading, setLoading] = useState(false)
     const [selectedLinks, setSelectedLinks] = useState([])
     const [url, setUrl] = useState('')
@@ -53,9 +64,38 @@ const ManageScrapedLinksDialog = ({ show, dialogProps, onCancel, onSave }) => {
 
     const handleFetchLinks = async () => {
         setLoading(true)
-        const fetchLinksResp = await scraperApi.fetchAllLinks(url, 'webCrawl')
-        if (fetchLinksResp.data) {
-            setSelectedLinks(fetchLinksResp.data.links)
+        try {
+            const fetchLinksResp = await scraperApi.fetchLinks(url, dialogProps.relativeLinksMethod, dialogProps.limit)
+            if (fetchLinksResp.data) {
+                setSelectedLinks(fetchLinksResp.data.links)
+                enqueueSnackbar({
+                    message: 'Successfully fetched links',
+                    options: {
+                        key: new Date().getTime() + Math.random(),
+                        variant: 'success',
+                        action: (key) => (
+                            <Button style={{ color: 'white' }} onClick={() => closeSnackbar(key)}>
+                                <IconX />
+                            </Button>
+                        )
+                    }
+                })
+            }
+        } catch (error) {
+            const errorData = error.response.data || `${error.response.status}: ${error.response.statusText}`
+            enqueueSnackbar({
+                message: errorData,
+                options: {
+                    key: new Date().getTime() + Math.random(),
+                    variant: 'error',
+                    persist: true,
+                    action: (key) => (
+                        <Button style={{ color: 'white' }} onClick={() => closeSnackbar(key)}>
+                            <IconX />
+                        </Button>
+                    )
+                }
+            })
         }
         setLoading(false)
     }

packages/ui/src/views/canvas/NodeInputHandler.js

@@ -91,9 +91,11 @@ const NodeInputHandler = ({ inputAnchor, inputParam, data, disabled = false, isA
         }
     }
 
-    const onManageLinksDialogClicked = (url, selectedLinks) => {
+    const onManageLinksDialogClicked = (url, selectedLinks, relativeLinksMethod, limit) => {
         const dialogProps = {
             url,
+            relativeLinksMethod,
+            limit,
             selectedLinks,
             confirmButtonName: 'Save',
             cancelButtonName: 'Cancel'
@@ -475,7 +477,9 @@ const NodeInputHandler = ({ inputAnchor, inputParam, data, disabled = false, isA
                                         onClick={() =>
                                             onManageLinksDialogClicked(
                                                 data.inputs[inputParam.name] ?? inputParam.default ?? '',
-                                                data.inputs.selectedLinks
+                                                data.inputs.selectedLinks,
+                                                data.inputs['relativeLinksMethod'] ?? 'webCrawl',
+                                                parseInt(data.inputs['limit']) ?? 0
                                             )
                                         }
                                     >