diff --git a/packages/components/nodes/documentloaders/SerpApi/SerpAPI.ts b/packages/components/nodes/documentloaders/SerpApi/SerpAPI.ts new file mode 100644 index 00000000..1d26400a --- /dev/null +++ b/packages/components/nodes/documentloaders/SerpApi/SerpAPI.ts @@ -0,0 +1,82 @@ +import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'; +import { TextSplitter } from 'langchain/text_splitter'; +import { SerpAPILoader } from 'langchain/document_loaders/web/serpapi';import { getCredentialData, getCredentialParam } from '../../../src'; + +class SerpAPI_DocumentLoaders implements INode { + label: string; + name: string; + version: number; + description: string; + type: string; + icon: string; + category: string; + baseClasses: string[]; + credential: INodeParams; + inputs: INodeParams[]; + + constructor() { + this.label = 'SerpApi For Web Search'; + this.name = 'serpApi'; + this.version = 1.0; + this.type = 'Document'; + this.icon = 'serp.png'; + this.category = 'Document Loaders'; + this.description = 'Load and process data from web search results'; + this.baseClasses = [this.type]; + this.credential = { + label: 'Connect Credential', + name: 'credential', + type: 'credential', + optional: false, + credentialNames: ['serpApi'] + }; + this.inputs = [ + { + label: 'Query', + name: 'query', + type: 'string' + }, + { + label: 'Text Splitter', + name: 'textSplitter', + type: 'TextSplitter', + optional: true + }, + { + label: 'Metadata', + name: 'metadata', + type: 'json', + optional: true, + additionalParams: true + } + ]; + } + + async init(nodeData: INodeData, _: string, options: ICommonObject): Promise { + const textSplitter = nodeData.inputs?.textSplitter as TextSplitter; + const query = nodeData.inputs?.query as string; + const metadata = nodeData.inputs?.metadata; + + const credentialData = await getCredentialData(nodeData.credential ?? '', options); + const serpApiKey = getCredentialParam('serpApiKey', credentialData, nodeData); + const loader = new SerpAPILoader({ q: query, apiKey: serpApiKey }); + const docs = textSplitter ? await loader.loadAndSplit() : await loader.load(); + + if (metadata) { + const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata); + return docs.map((doc) => { + return { + ...doc, + metadata: { + ...doc.metadata, + ...parsedMetadata + } + }; + }); + } + + return docs; + } +} + +module.exports = { nodeClass: SerpAPI_DocumentLoaders }; diff --git a/packages/components/nodes/documentloaders/SerpApi/serp.png b/packages/components/nodes/documentloaders/SerpApi/serp.png new file mode 100644 index 00000000..338aeaea Binary files /dev/null and b/packages/components/nodes/documentloaders/SerpApi/serp.png differ diff --git a/packages/server/marketplaces/chatflows/Web Search QnA.json b/packages/server/marketplaces/chatflows/Web Search QnA.json new file mode 100644 index 00000000..d604437e --- /dev/null +++ b/packages/server/marketplaces/chatflows/Web Search QnA.json @@ -0,0 +1,596 @@ +{ + "nodes": [ + { + "width": 300, + "height": 525, + "id": "chatOpenAI_0", + "position": { + "x": 858.6611234371157, + "y": 40.225188014879166 + }, + "type": "customNode", + "data": { + "id": "chatOpenAI_0", + "label": "ChatOpenAI", + "version": 1, + "name": "chatOpenAI", + "type": "ChatOpenAI", + "baseClasses": [ + "ChatOpenAI", + "BaseChatModel", + "BaseLanguageModel", + "Runnable" + ], + "category": "Chat Models", + "description": "Wrapper around OpenAI large language models that use the Chat endpoint", + "inputParams": [ + { + "label": "Connect Credential", + "name": "credential", + "type": "credential", + "credentialNames": [ + "openAIApi" + ], + "id": "chatOpenAI_0-input-credential-credential" + }, + { + "label": "Model Name", + "name": "modelName", + "type": "options", + "options": [ + { + "label": "gpt-4", + "name": "gpt-4" + }, + { + "label": "gpt-4-0613", + "name": "gpt-4-0613" + }, + { + "label": "gpt-4-32k", + "name": "gpt-4-32k" + }, + { + "label": "gpt-4-32k-0613", + "name": "gpt-4-32k-0613" + }, + { + "label": "gpt-3.5-turbo", + "name": "gpt-3.5-turbo" + }, + { + "label": "gpt-3.5-turbo-0613", + "name": "gpt-3.5-turbo-0613" + }, + { + "label": "gpt-3.5-turbo-16k", + "name": "gpt-3.5-turbo-16k" + }, + { + "label": "gpt-3.5-turbo-16k-0613", + "name": "gpt-3.5-turbo-16k-0613" + } + ], + "default": "gpt-3.5-turbo", + "optional": true, + "id": "chatOpenAI_0-input-modelName-options" + }, + { + "label": "Temperature", + "name": "temperature", + "type": "number", + "step": 0.1, + "default": 0.9, + "optional": true, + "id": "chatOpenAI_0-input-temperature-number" + }, + { + "label": "Max Tokens", + "name": "maxTokens", + "type": "number", + "step": 1, + "optional": true, + "additionalParams": true, + "id": "chatOpenAI_0-input-maxTokens-number" + }, + { + "label": "Top Probability", + "name": "topP", + "type": "number", + "step": 0.1, + "optional": true, + "additionalParams": true, + "id": "chatOpenAI_0-input-topP-number" + }, + { + "label": "Frequency Penalty", + "name": "frequencyPenalty", + "type": "number", + "step": 0.1, + "optional": true, + "additionalParams": true, + "id": "chatOpenAI_0-input-frequencyPenalty-number" + }, + { + "label": "Presence Penalty", + "name": "presencePenalty", + "type": "number", + "step": 0.1, + "optional": true, + "additionalParams": true, + "id": "chatOpenAI_0-input-presencePenalty-number" + }, + { + "label": "Timeout", + "name": "timeout", + "type": "number", + "step": 1, + "optional": true, + "additionalParams": true, + "id": "chatOpenAI_0-input-timeout-number" + }, + { + "label": "BasePath", + "name": "basepath", + "type": "string", + "optional": true, + "additionalParams": true, + "id": "chatOpenAI_0-input-basepath-string" + }, + { + "label": "BaseOptions", + "name": "baseOptions", + "type": "json", + "optional": true, + "additionalParams": true, + "id": "chatOpenAI_0-input-baseOptions-json" + } + ], + "inputAnchors": [], + "inputs": { + "modelName": "gpt-4", + "temperature": 0.9, + "maxTokens": "", + "topP": "", + "frequencyPenalty": "", + "presencePenalty": "", + "timeout": "", + "basepath": "", + "baseOptions": "" + }, + "outputAnchors": [ + { + "id": "chatOpenAI_0-output-chatOpenAI-ChatOpenAI|BaseChatModel|BaseLanguageModel|Runnable", + "name": "chatOpenAI", + "label": "ChatOpenAI", + "type": "ChatOpenAI | BaseChatModel | BaseLanguageModel | Runnable" + } + ], + "outputs": {}, + "selected": false + }, + "selected": false, + "positionAbsolute": { + "x": 858.6611234371157, + "y": 40.225188014879166 + }, + "dragging": false + }, + { + "width": 300, + "height": 481, + "id": "conversationalRetrievalQAChain_0", + "position": { + "x": 1584.2201037703737, + "y": 207.80548502487522 + }, + "type": "customNode", + "data": { + "id": "conversationalRetrievalQAChain_0", + "label": "Conversational Retrieval QA Chain", + "version": 1, + "name": "conversationalRetrievalQAChain", + "type": "ConversationalRetrievalQAChain", + "baseClasses": [ + "ConversationalRetrievalQAChain", + "BaseChain", + "Runnable" + ], + "category": "Chains", + "description": "Document QA - built on RetrievalQAChain to provide a chat history component", + "inputParams": [ + { + "label": "Return Source Documents", + "name": "returnSourceDocuments", + "type": "boolean", + "optional": true, + "id": "conversationalRetrievalQAChain_0-input-returnSourceDocuments-boolean" + }, + { + "label": "System Message", + "name": "systemMessagePrompt", + "type": "string", + "rows": 4, + "additionalParams": true, + "optional": true, + "placeholder": "I want you to act as a document that I am having a conversation with. Your name is \"AI Assistant\". You will provide me with answers from the given info. If the answer is not included, say exactly \"Hmm, I am not sure.\" and stop after that. Refuse to answer any question not about the info. Never break character.", + "id": "conversationalRetrievalQAChain_0-input-systemMessagePrompt-string" + }, + { + "label": "Chain Option", + "name": "chainOption", + "type": "options", + "options": [ + { + "label": "MapReduceDocumentsChain", + "name": "map_reduce", + "description": "Suitable for QA tasks over larger documents and can run the preprocessing step in parallel, reducing the running time" + }, + { + "label": "RefineDocumentsChain", + "name": "refine", + "description": "Suitable for QA tasks over a large number of documents." + }, + { + "label": "StuffDocumentsChain", + "name": "stuff", + "description": "Suitable for QA tasks over a small number of documents." + } + ], + "additionalParams": true, + "optional": true, + "id": "conversationalRetrievalQAChain_0-input-chainOption-options" + } + ], + "inputAnchors": [ + { + "label": "Language Model", + "name": "model", + "type": "BaseLanguageModel", + "id": "conversationalRetrievalQAChain_0-input-model-BaseLanguageModel" + }, + { + "label": "Vector Store Retriever", + "name": "vectorStoreRetriever", + "type": "BaseRetriever", + "id": "conversationalRetrievalQAChain_0-input-vectorStoreRetriever-BaseRetriever" + }, + { + "label": "Memory", + "name": "memory", + "type": "BaseMemory", + "optional": true, + "description": "If left empty, a default BufferMemory will be used", + "id": "conversationalRetrievalQAChain_0-input-memory-BaseMemory" + } + ], + "inputs": { + "model": "{{chatOpenAI_0.data.instance}}", + "vectorStoreRetriever": "{{memoryVectorStore_0.data.instance}}", + "memory": "", + "returnSourceDocuments": true, + "systemMessagePrompt": "", + "chainOption": "" + }, + "outputAnchors": [ + { + "id": "conversationalRetrievalQAChain_0-output-conversationalRetrievalQAChain-ConversationalRetrievalQAChain|BaseChain|Runnable", + "name": "conversationalRetrievalQAChain", + "label": "ConversationalRetrievalQAChain", + "type": "ConversationalRetrievalQAChain | BaseChain | Runnable" + } + ], + "outputs": {}, + "selected": false + }, + "selected": false, + "positionAbsolute": { + "x": 1584.2201037703737, + "y": 207.80548502487522 + }, + "dragging": false + }, + { + "width": 300, + "height": 330, + "id": "openAIEmbeddings_0", + "position": { + "x": 59.99572465636521, + "y": 789.1458487580028 + }, + "type": "customNode", + "data": { + "id": "openAIEmbeddings_0", + "label": "OpenAI Embeddings", + "version": 1, + "name": "openAIEmbeddings", + "type": "OpenAIEmbeddings", + "baseClasses": [ + "OpenAIEmbeddings", + "Embeddings" + ], + "category": "Embeddings", + "description": "OpenAI API to generate embeddings for a given text", + "inputParams": [ + { + "label": "Connect Credential", + "name": "credential", + "type": "credential", + "credentialNames": [ + "openAIApi" + ], + "id": "openAIEmbeddings_0-input-credential-credential" + }, + { + "label": "Strip New Lines", + "name": "stripNewLines", + "type": "boolean", + "optional": true, + "additionalParams": true, + "id": "openAIEmbeddings_0-input-stripNewLines-boolean" + }, + { + "label": "Batch Size", + "name": "batchSize", + "type": "number", + "optional": true, + "additionalParams": true, + "id": "openAIEmbeddings_0-input-batchSize-number" + }, + { + "label": "Timeout", + "name": "timeout", + "type": "number", + "optional": true, + "additionalParams": true, + "id": "openAIEmbeddings_0-input-timeout-number" + }, + { + "label": "BasePath", + "name": "basepath", + "type": "string", + "optional": true, + "additionalParams": true, + "id": "openAIEmbeddings_0-input-basepath-string" + } + ], + "inputAnchors": [], + "inputs": { + "stripNewLines": "", + "batchSize": "", + "timeout": "", + "basepath": "" + }, + "outputAnchors": [ + { + "id": "openAIEmbeddings_0-output-openAIEmbeddings-OpenAIEmbeddings|Embeddings", + "name": "openAIEmbeddings", + "label": "OpenAIEmbeddings", + "type": "OpenAIEmbeddings | Embeddings" + } + ], + "outputs": {}, + "selected": false + }, + "selected": false, + "positionAbsolute": { + "x": 59.99572465636521, + "y": 789.1458487580028 + }, + "dragging": false + }, + { + "width": 300, + "height": 407, + "id": "memoryVectorStore_0", + "position": { + "x": 804.8405953555847, + "y": 780.514305397237 + }, + "type": "customNode", + "data": { + "id": "memoryVectorStore_0", + "label": "In-Memory Vector Store", + "version": 1, + "name": "memoryVectorStore", + "type": "Memory", + "baseClasses": [ + "Memory", + "VectorStoreRetriever", + "BaseRetriever" + ], + "category": "Vector Stores", + "description": "In-memory vectorstore that stores embeddings and does an exact, linear search for the most similar embeddings.", + "inputParams": [ + { + "label": "Top K", + "name": "topK", + "description": "Number of top results to fetch. Default to 4", + "placeholder": "4", + "type": "number", + "optional": true, + "id": "memoryVectorStore_0-input-topK-number" + } + ], + "inputAnchors": [ + { + "label": "Document", + "name": "document", + "type": "Document", + "list": true, + "id": "memoryVectorStore_0-input-document-Document" + }, + { + "label": "Embeddings", + "name": "embeddings", + "type": "Embeddings", + "id": "memoryVectorStore_0-input-embeddings-Embeddings" + } + ], + "inputs": { + "document": [ + "{{serpApi_2.data.instance}}" + ], + "embeddings": "{{openAIEmbeddings_0.data.instance}}", + "topK": "" + }, + "outputAnchors": [ + { + "name": "output", + "label": "Output", + "type": "options", + "options": [ + { + "id": "memoryVectorStore_0-output-retriever-Memory|VectorStoreRetriever|BaseRetriever", + "name": "retriever", + "label": "Memory Retriever", + "type": "Memory | VectorStoreRetriever | BaseRetriever" + }, + { + "id": "memoryVectorStore_0-output-vectorStore-Memory|VectorStore", + "name": "vectorStore", + "label": "Memory Vector Store", + "type": "Memory | VectorStore" + } + ], + "default": "retriever" + } + ], + "outputs": { + "output": "retriever" + }, + "selected": false + }, + "positionAbsolute": { + "x": 804.8405953555847, + "y": 780.514305397237 + }, + "selected": false, + "dragging": false + }, + { + "width": 300, + "height": 480, + "id": "serpApi_2", + "position": { + "x": 199.84883781665212, + "y": 113.02413239018131 + }, + "type": "customNode", + "data": { + "id": "serpApi_2", + "label": "SerpApi For Web Search", + "version": 1, + "name": "serpApi", + "type": "Document", + "baseClasses": [ + "Document" + ], + "category": "Document Loaders", + "description": "Load and process data from web search results", + "inputParams": [ + { + "label": "Connect Credential", + "name": "credential", + "type": "credential", + "optional": false, + "credentialNames": [ + "serpApi" + ], + "id": "serpApi_2-input-credential-credential" + }, + { + "label": "Query", + "name": "query", + "type": "string", + "id": "serpApi_2-input-query-string" + }, + { + "label": "Metadata", + "name": "metadata", + "type": "json", + "optional": true, + "additionalParams": true, + "id": "serpApi_2-input-metadata-json" + } + ], + "inputAnchors": [ + { + "label": "Text Splitter", + "name": "textSplitter", + "type": "TextSplitter", + "optional": true, + "id": "serpApi_2-input-textSplitter-TextSplitter" + } + ], + "inputs": { + "query": "Coffee", + "textSplitter": "", + "metadata": "" + }, + "outputAnchors": [ + { + "id": "serpApi_2-output-serpApi-Document", + "name": "serpApi", + "label": "Document", + "type": "Document" + } + ], + "outputs": {}, + "selected": false + }, + "selected": false, + "positionAbsolute": { + "x": 199.84883781665212, + "y": 113.02413239018131 + }, + "dragging": false + } + ], + "edges": [ + { + "source": "openAIEmbeddings_0", + "sourceHandle": "openAIEmbeddings_0-output-openAIEmbeddings-OpenAIEmbeddings|Embeddings", + "target": "memoryVectorStore_0", + "targetHandle": "memoryVectorStore_0-input-embeddings-Embeddings", + "type": "buttonedge", + "id": "openAIEmbeddings_0-openAIEmbeddings_0-output-openAIEmbeddings-OpenAIEmbeddings|Embeddings-memoryVectorStore_0-memoryVectorStore_0-input-embeddings-Embeddings", + "data": { + "label": "" + } + }, + { + "source": "memoryVectorStore_0", + "sourceHandle": "memoryVectorStore_0-output-retriever-Memory|VectorStoreRetriever|BaseRetriever", + "target": "conversationalRetrievalQAChain_0", + "targetHandle": "conversationalRetrievalQAChain_0-input-vectorStoreRetriever-BaseRetriever", + "type": "buttonedge", + "id": "memoryVectorStore_0-memoryVectorStore_0-output-retriever-Memory|VectorStoreRetriever|BaseRetriever-conversationalRetrievalQAChain_0-conversationalRetrievalQAChain_0-input-vectorStoreRetriever-BaseRetriever", + "data": { + "label": "" + } + }, + { + "source": "chatOpenAI_0", + "sourceHandle": "chatOpenAI_0-output-chatOpenAI-ChatOpenAI|BaseChatModel|BaseLanguageModel|Runnable", + "target": "conversationalRetrievalQAChain_0", + "targetHandle": "conversationalRetrievalQAChain_0-input-model-BaseLanguageModel", + "type": "buttonedge", + "id": "chatOpenAI_0-chatOpenAI_0-output-chatOpenAI-ChatOpenAI|BaseChatModel|BaseLanguageModel|Runnable-conversationalRetrievalQAChain_0-conversationalRetrievalQAChain_0-input-model-BaseLanguageModel", + "data": { + "label": "" + } + }, + { + "source": "serpApi_2", + "sourceHandle": "serpApi_2-output-serpApi-Document", + "target": "memoryVectorStore_0", + "targetHandle": "memoryVectorStore_0-input-document-Document", + "type": "buttonedge", + "id": "serpApi_2-serpApi_2-output-serpApi-Document-memoryVectorStore_0-memoryVectorStore_0-input-document-Document", + "data": { + "label": "" + } + } + ] +} \ No newline at end of file