diff --git a/packages/components/nodes/documentloaders/Cheerio/Cheerio.ts b/packages/components/nodes/documentloaders/Cheerio/Cheerio.ts new file mode 100644 index 00000000..8be10f74 --- /dev/null +++ b/packages/components/nodes/documentloaders/Cheerio/Cheerio.ts @@ -0,0 +1,64 @@ +import { INode, INodeData, INodeParams } from '../../../src/Interface' +import { TextSplitter } from 'langchain/text_splitter' +import { CheerioWebBaseLoader } from 'langchain/document_loaders/web/cheerio' + +class Cheerio_DocumentLoaders implements INode { + label: string + name: string + description: string + type: string + icon: string + category: string + baseClasses: string[] + inputs: INodeParams[] + + constructor() { + this.label = 'Cheerio Web Scraper' + this.name = 'cheerioWebScraper' + this.type = 'Document' + this.icon = 'cheerio.svg' + this.category = 'Document Loaders' + this.description = `Load data from webpages` + this.baseClasses = [this.type] + this.inputs = [ + { + label: 'URL', + name: 'url', + type: 'string' + }, + { + label: 'Text Splitter', + name: 'textSplitter', + type: 'TextSplitter', + optional: true + } + ] + } + + async init(nodeData: INodeData): Promise { + const textSplitter = nodeData.inputs?.textSplitter as TextSplitter + let url = nodeData.inputs?.url as string + + var urlPattern = new RegExp( + '^(https?:\\/\\/)?' + // validate protocol + '((([a-z\\d]([a-z\\d-]*[a-z\\d])*)\\.)+[a-z]{2,}|' + // validate domain name + '((\\d{1,3}\\.){3}\\d{1,3}))' + // validate OR ip (v4) address + '(\\:\\d+)?(\\/[-a-z\\d%_.~+]*)*' + // validate port and path + '(\\?[;&a-z\\d%_.~+=-]*)?' + // validate query string + '(\\#[-a-z\\d_]*)?$', + 'i' + ) // validate fragment locator + + const loader = new CheerioWebBaseLoader(urlPattern.test(url.trim()) ? url.trim() : '') + + if (textSplitter) { + const docs = await loader.loadAndSplit(textSplitter) + return docs + } else { + const docs = await loader.load() + return docs + } + } +} + +module.exports = { nodeClass: Cheerio_DocumentLoaders } diff --git a/packages/components/nodes/documentloaders/Cheerio/cheerio.svg b/packages/components/nodes/documentloaders/Cheerio/cheerio.svg new file mode 100644 index 00000000..8e3334b9 --- /dev/null +++ b/packages/components/nodes/documentloaders/Cheerio/cheerio.svg @@ -0,0 +1,5 @@ + + + + + \ No newline at end of file diff --git a/packages/components/nodes/documentloaders/Csv/Csv.png b/packages/components/nodes/documentloaders/Csv/Csv.png new file mode 100644 index 00000000..41b84e16 Binary files /dev/null and b/packages/components/nodes/documentloaders/Csv/Csv.png differ diff --git a/packages/components/nodes/documentloaders/Csv/Csv.ts b/packages/components/nodes/documentloaders/Csv/Csv.ts new file mode 100644 index 00000000..2ead7055 --- /dev/null +++ b/packages/components/nodes/documentloaders/Csv/Csv.ts @@ -0,0 +1,68 @@ +import { INode, INodeData, INodeParams } from '../../../src/Interface' +import { TextSplitter } from 'langchain/text_splitter' +import { CSVLoader } from 'langchain/document_loaders/fs/csv' + +class Csv_DocumentLoaders implements INode { + label: string + name: string + description: string + type: string + icon: string + category: string + baseClasses: string[] + inputs: INodeParams[] + + constructor() { + this.label = 'Csv File' + this.name = 'csvFile' + this.type = 'Document' + this.icon = 'Csv.png' + this.category = 'Document Loaders' + this.description = `Load data from CSV files` + this.baseClasses = [this.type] + this.inputs = [ + { + label: 'Csv File', + name: 'csvFile', + type: 'file', + fileType: '.csv' + }, + { + label: 'Text Splitter', + name: 'textSplitter', + type: 'TextSplitter', + optional: true + }, + { + label: 'Single Column Extraction', + name: 'columnName', + type: 'string', + description: 'Extracting a single column', + placeholder: 'Enter column name', + optional: true + } + ] + } + + async init(nodeData: INodeData): Promise { + const textSplitter = nodeData.inputs?.textSplitter as TextSplitter + const csvFileBase64 = nodeData.inputs?.csvFile as string + const columnName = nodeData.inputs?.columnName as string + const splitDataURI = csvFileBase64.split(',') + splitDataURI.pop() + const bf = Buffer.from(splitDataURI.pop() || '', 'base64') + + const blob = new Blob([bf]) + const loader = new CSVLoader(blob, columnName.trim().length === 0 ? undefined : columnName.trim()) + + if (textSplitter) { + const docs = await loader.loadAndSplit(textSplitter) + return docs + } else { + const docs = await loader.load() + return docs + } + } +} + +module.exports = { nodeClass: Csv_DocumentLoaders } diff --git a/packages/components/nodes/embeddings/CohereEmbedding/CohereEmbedding.ts b/packages/components/nodes/embeddings/CohereEmbedding/CohereEmbedding.ts new file mode 100644 index 00000000..923cf6c6 --- /dev/null +++ b/packages/components/nodes/embeddings/CohereEmbedding/CohereEmbedding.ts @@ -0,0 +1,40 @@ +import { INode, INodeData, INodeParams } from '../../../src/Interface' +import { getBaseClasses } from '../../../src/utils' +import { CohereEmbeddings } from 'langchain/embeddings/cohere' + +class CohereEmbedding_Embeddings implements INode { + label: string + name: string + type: string + icon: string + category: string + description: string + baseClasses: string[] + inputs: INodeParams[] + + constructor() { + this.label = 'Cohere Embeddings' + this.name = 'cohereEmbeddings' + this.type = 'CohereEmbeddings' + this.icon = 'cohere.png' + this.category = 'Embeddings' + this.description = 'Cohere API to generate embeddings for a given text' + this.baseClasses = [this.type, ...getBaseClasses(CohereEmbeddings)] + this.inputs = [ + { + label: 'Cohere API Key', + name: 'cohereApiKey', + type: 'password' + } + ] + } + + async init(nodeData: INodeData): Promise { + const apiKey = nodeData.inputs?.cohereApiKey as string + + const model = new CohereEmbeddings({ apiKey }) + return model + } +} + +module.exports = { nodeClass: CohereEmbedding_Embeddings } diff --git a/packages/components/nodes/embeddings/CohereEmbedding/cohere.png b/packages/components/nodes/embeddings/CohereEmbedding/cohere.png new file mode 100644 index 00000000..266adeac Binary files /dev/null and b/packages/components/nodes/embeddings/CohereEmbedding/cohere.png differ diff --git a/packages/components/package.json b/packages/components/package.json index d2d68037..0e845475 100644 --- a/packages/components/package.json +++ b/packages/components/package.json @@ -20,7 +20,10 @@ "@huggingface/inference": "^1.6.3", "@pinecone-database/pinecone": "^0.0.12", "axios": "^0.27.2", + "cheerio": "^1.0.0-rc.12", "chromadb": "^1.3.1", + "cohere-ai": "^6.2.0", + "d3-dsv": "2", "dotenv": "^16.0.0", "express": "^4.17.3", "form-data": "^4.0.0", diff --git a/packages/ui/src/store/context/ReactFlowContext.js b/packages/ui/src/store/context/ReactFlowContext.js index 66a08397..bbf4c0d4 100644 --- a/packages/ui/src/store/context/ReactFlowContext.js +++ b/packages/ui/src/store/context/ReactFlowContext.js @@ -16,13 +16,51 @@ export const flowContext = createContext(initialValue) export const ReactFlowContext = ({ children }) => { const [reactFlowInstance, setReactFlowInstance] = useState(null) - const deleteNode = (id) => { - reactFlowInstance.setNodes(reactFlowInstance.getNodes().filter((n) => n.id !== id)) - reactFlowInstance.setEdges(reactFlowInstance.getEdges().filter((ns) => ns.source !== id && ns.target !== id)) + const deleteNode = (nodeid) => { + deleteConnectedInput(nodeid, 'node') + reactFlowInstance.setNodes(reactFlowInstance.getNodes().filter((n) => n.id !== nodeid)) + reactFlowInstance.setEdges(reactFlowInstance.getEdges().filter((ns) => ns.source !== nodeid && ns.target !== nodeid)) } - const deleteEdge = (id) => { - reactFlowInstance.setEdges(reactFlowInstance.getEdges().filter((edge) => edge.id !== id)) + const deleteEdge = (edgeid) => { + deleteConnectedInput(edgeid, 'edge') + reactFlowInstance.setEdges(reactFlowInstance.getEdges().filter((edge) => edge.id !== edgeid)) + } + + const deleteConnectedInput = (id, type) => { + const connectedEdges = + type === 'node' + ? reactFlowInstance.getEdges().filter((edge) => edge.source === id) + : reactFlowInstance.getEdges().filter((edge) => edge.id === id) + + for (const edge of connectedEdges) { + const targetNodeId = edge.target + const sourceNodeId = edge.source + const targetInput = edge.targetHandle.split('-')[2] + + reactFlowInstance.setNodes((nds) => + nds.map((node) => { + if (node.id === targetNodeId) { + let value + const inputAnchor = node.data.inputAnchors.find((ancr) => ancr.name === targetInput) + if (inputAnchor && inputAnchor.list) { + const values = node.data.inputs[targetInput] || [] + value = values.filter((item) => !item.includes(sourceNodeId)) + } else { + value = '' + } + node.data = { + ...node.data, + inputs: { + ...node.data.inputs, + [targetInput]: value + } + } + } + return node + }) + ) + } } const duplicateNode = (id) => { diff --git a/packages/ui/src/views/chatmessage/ChatMessage.js b/packages/ui/src/views/chatmessage/ChatMessage.js index 4fbb54e6..b6632b1e 100644 --- a/packages/ui/src/views/chatmessage/ChatMessage.js +++ b/packages/ui/src/views/chatmessage/ChatMessage.js @@ -39,6 +39,7 @@ export const ChatMessage = ({ chatflowid }) => { const customization = useSelector((state) => state.customization) const { confirm } = useConfirm() const dispatch = useDispatch() + const ps = useRef() useNotifier() const enqueueSnackbar = (...args) => dispatch(enqueueSnackbarAction(...args)) @@ -54,7 +55,6 @@ export const ChatMessage = ({ chatflowid }) => { } ]) - const messagesEndRef = useRef(null) const inputRef = useRef(null) const anchorRef = useRef(null) const prevOpen = useRef(open) @@ -115,7 +115,9 @@ export const ChatMessage = ({ chatflowid }) => { } const scrollToBottom = () => { - messagesEndRef.current?.scrollIntoView({ behavior: 'smooth' }) + if (ps.current) { + ps.current.scrollTo({ top: Number.MAX_SAFE_INTEGER, behavior: 'smooth' }) + } } const addChatMessage = async (message, type) => { @@ -286,7 +288,7 @@ export const ChatMessage = ({ chatflowid }) => {
-
+
{messages.map((message, index) => { return ( // The latest message sent by the user will be animated while waiting for a response @@ -331,7 +333,6 @@ export const ChatMessage = ({ chatflowid }) => { ) })} -