From 23d39300cb9b2899572294abbb1c3a5c4010d26b Mon Sep 17 00:00:00 2001 From: Ziyi Chen Date: Tue, 24 Oct 2023 11:04:35 -0400 Subject: [PATCH 01/12] fix Cannot read properties of undefined (reading 'replace') --- .../nodes/vectorstores/InMemory/InMemoryVectorStore.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/components/nodes/vectorstores/InMemory/InMemoryVectorStore.ts b/packages/components/nodes/vectorstores/InMemory/InMemoryVectorStore.ts index 55a01e2b..fd2246c9 100644 --- a/packages/components/nodes/vectorstores/InMemory/InMemoryVectorStore.ts +++ b/packages/components/nodes/vectorstores/InMemory/InMemoryVectorStore.ts @@ -71,7 +71,9 @@ class InMemoryVectorStore_VectorStores implements INode { const flattenDocs = docs && docs.length ? flatten(docs) : [] const finalDocs = [] for (let i = 0; i < flattenDocs.length; i += 1) { - finalDocs.push(new Document(flattenDocs[i])) + if (flattenDocs[i] !== undefined) { + finalDocs.push(new Document(flattenDocs[i])) + } } const vectorStore = await MemoryVectorStore.fromDocuments(finalDocs, embeddings) From 6c7734302d2d6257f133e9535b025a7aab7339d0 Mon Sep 17 00:00:00 2001 From: Henry Date: Wed, 25 Oct 2023 17:34:24 +0100 Subject: [PATCH 02/12] fix redis chat sessionId --- .../RedisBackedChatMemory/RedisBackedChatMemory.ts | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/packages/components/nodes/memory/RedisBackedChatMemory/RedisBackedChatMemory.ts b/packages/components/nodes/memory/RedisBackedChatMemory/RedisBackedChatMemory.ts index 3e3697d1..c65d729b 100644 --- a/packages/components/nodes/memory/RedisBackedChatMemory/RedisBackedChatMemory.ts +++ b/packages/components/nodes/memory/RedisBackedChatMemory/RedisBackedChatMemory.ts @@ -119,27 +119,26 @@ const initalizeRedis = async (nodeData: INodeData, options: ICommonObject): Prom const redisChatMessageHistory = new RedisChatMessageHistory(obj) redisChatMessageHistory.getMessages = async (): Promise => { - const rawStoredMessages = await client.lrange(sessionId ? sessionId : chatId, 0, -1) + const rawStoredMessages = await client.lrange((redisChatMessageHistory as any).sessionId, 0, -1) const orderedMessages = rawStoredMessages.reverse().map((message) => JSON.parse(message)) return orderedMessages.map(mapStoredMessageToChatMessage) } redisChatMessageHistory.addMessage = async (message: BaseMessage): Promise => { const messageToAdd = [message].map((msg) => msg.toDict()) - await client.lpush(sessionId ? sessionId : chatId, JSON.stringify(messageToAdd[0])) + await client.lpush((redisChatMessageHistory as any).sessionId, JSON.stringify(messageToAdd[0])) if (sessionTTL) { - await client.expire(sessionId ? sessionId : chatId, sessionTTL) + await client.expire((redisChatMessageHistory as any).sessionId, sessionTTL) } } redisChatMessageHistory.clear = async (): Promise => { - await client.del(sessionId ? sessionId : chatId) + await client.del((redisChatMessageHistory as any).sessionId) } const memory = new BufferMemoryExtended({ memoryKey, chatHistory: redisChatMessageHistory, - returnMessages: true, isSessionIdUsingChatMessageId }) return memory From cd6538c5f2acffb5f018523eec11802389be96cd Mon Sep 17 00:00:00 2001 From: vinodkiran Date: Thu, 26 Oct 2023 11:11:10 +0530 Subject: [PATCH 03/12] Spelling Corrections. --- packages/components/nodes/agents/CSVAgent/CSVAgent.ts | 2 +- .../nodes/memory/BufferWindowMemory/BufferWindowMemory.ts | 4 ++-- packages/components/nodes/memory/ZepMemory/ZepMemory.ts | 2 +- .../prompts/FewShotPromptTemplate/FewShotPromptTemplate.ts | 2 +- .../CharacterTextSplitter/CharacterTextSplitter.ts | 2 +- .../RecursiveCharacterTextSplitter.ts | 2 +- packages/components/src/utils.ts | 4 ++-- packages/server/marketplaces/chatflows/Antonym.json | 2 +- 8 files changed, 10 insertions(+), 10 deletions(-) diff --git a/packages/components/nodes/agents/CSVAgent/CSVAgent.ts b/packages/components/nodes/agents/CSVAgent/CSVAgent.ts index 905bef02..5baad2ec 100644 --- a/packages/components/nodes/agents/CSVAgent/CSVAgent.ts +++ b/packages/components/nodes/agents/CSVAgent/CSVAgent.ts @@ -131,7 +131,7 @@ json.dumps(my_dict)` const code = `import pandas as pd\n${pythonCode}` finalResult = await pyodide.runPythonAsync(code) } catch (error) { - throw new Error(`Sorry, I'm unable to find answer for question: "${input}" using follwoing code: "${pythonCode}"`) + throw new Error(`Sorry, I'm unable to find answer for question: "${input}" using following code: "${pythonCode}"`) } } diff --git a/packages/components/nodes/memory/BufferWindowMemory/BufferWindowMemory.ts b/packages/components/nodes/memory/BufferWindowMemory/BufferWindowMemory.ts index cf8e7f1d..84e607e5 100644 --- a/packages/components/nodes/memory/BufferWindowMemory/BufferWindowMemory.ts +++ b/packages/components/nodes/memory/BufferWindowMemory/BufferWindowMemory.ts @@ -20,7 +20,7 @@ class BufferWindowMemory_Memory implements INode { this.type = 'BufferWindowMemory' this.icon = 'memory.svg' this.category = 'Memory' - this.description = 'Uses a window of size k to surface the last k back-and-forths to use as memory' + this.description = 'Uses a window of size k to surface the last k back-and-forth to use as memory' this.baseClasses = [this.type, ...getBaseClasses(BufferWindowMemory)] this.inputs = [ { @@ -40,7 +40,7 @@ class BufferWindowMemory_Memory implements INode { name: 'k', type: 'number', default: '4', - description: 'Window of size k to surface the last k back-and-forths to use as memory.' + description: 'Window of size k to surface the last k back-and-forth to use as memory.' } ] } diff --git a/packages/components/nodes/memory/ZepMemory/ZepMemory.ts b/packages/components/nodes/memory/ZepMemory/ZepMemory.ts index 0c05563a..c4498644 100644 --- a/packages/components/nodes/memory/ZepMemory/ZepMemory.ts +++ b/packages/components/nodes/memory/ZepMemory/ZepMemory.ts @@ -60,7 +60,7 @@ class ZepMemory_Memory implements INode { name: 'k', type: 'number', default: '10', - description: 'Window of size k to surface the last k back-and-forths to use as memory.' + description: 'Window of size k to surface the last k back-and-forth to use as memory.' }, { label: 'Auto Summary Template', diff --git a/packages/components/nodes/prompts/FewShotPromptTemplate/FewShotPromptTemplate.ts b/packages/components/nodes/prompts/FewShotPromptTemplate/FewShotPromptTemplate.ts index 482d8745..a1a2afc2 100644 --- a/packages/components/nodes/prompts/FewShotPromptTemplate/FewShotPromptTemplate.ts +++ b/packages/components/nodes/prompts/FewShotPromptTemplate/FewShotPromptTemplate.ts @@ -55,7 +55,7 @@ class FewShotPromptTemplate_Prompts implements INode { placeholder: `Word: {input}\nAntonym:` }, { - label: 'Example Seperator', + label: 'Example Separator', name: 'exampleSeparator', type: 'string', placeholder: `\n\n` diff --git a/packages/components/nodes/textsplitters/CharacterTextSplitter/CharacterTextSplitter.ts b/packages/components/nodes/textsplitters/CharacterTextSplitter/CharacterTextSplitter.ts index ed972ad6..76c5cfb2 100644 --- a/packages/components/nodes/textsplitters/CharacterTextSplitter/CharacterTextSplitter.ts +++ b/packages/components/nodes/textsplitters/CharacterTextSplitter/CharacterTextSplitter.ts @@ -41,7 +41,7 @@ class CharacterTextSplitter_TextSplitters implements INode { name: 'separator', type: 'string', placeholder: `" "`, - description: 'Seperator to determine when to split the text, will override the default separator', + description: 'Separator to determine when to split the text, will override the default separator', optional: true } ] diff --git a/packages/components/nodes/textsplitters/RecursiveCharacterTextSplitter/RecursiveCharacterTextSplitter.ts b/packages/components/nodes/textsplitters/RecursiveCharacterTextSplitter/RecursiveCharacterTextSplitter.ts index 63f71267..d7dbc1c3 100644 --- a/packages/components/nodes/textsplitters/RecursiveCharacterTextSplitter/RecursiveCharacterTextSplitter.ts +++ b/packages/components/nodes/textsplitters/RecursiveCharacterTextSplitter/RecursiveCharacterTextSplitter.ts @@ -41,7 +41,7 @@ class RecursiveCharacterTextSplitter_TextSplitters implements INode { name: 'separators', type: 'string', rows: 4, - description: 'Array of custom seperators to determine when to split the text, will override the default separators', + description: 'Array of custom separators to determine when to split the text, will override the default separators', placeholder: `["|", "##", ">", "-"]`, optional: true } diff --git a/packages/components/src/utils.ts b/packages/components/src/utils.ts index 8f33683c..3c3c6ee2 100644 --- a/packages/components/src/utils.ts +++ b/packages/components/src/utils.ts @@ -301,7 +301,7 @@ async function crawl(baseURL: string, currentURL: string, pages: string[], limit } /** - * Prep URL before passing into recursive carwl function + * Prep URL before passing into recursive crawl function * @param {string} stringURL * @param {number} limit * @returns {Promise} @@ -445,7 +445,7 @@ export const getCredentialData = async (selectedCredentialId: string, options: I if (!credential) return {} - // Decrpyt credentialData + // Decrypt credentialData const decryptedCredentialData = await decryptCredentialData(credential.encryptedData) return decryptedCredentialData diff --git a/packages/server/marketplaces/chatflows/Antonym.json b/packages/server/marketplaces/chatflows/Antonym.json index 5f8ff7a8..54342d8f 100644 --- a/packages/server/marketplaces/chatflows/Antonym.json +++ b/packages/server/marketplaces/chatflows/Antonym.json @@ -45,7 +45,7 @@ "id": "fewShotPromptTemplate_1-input-suffix-string" }, { - "label": "Example Seperator", + "label": "Example Separator", "name": "exampleSeparator", "type": "string", "placeholder": "\n\n", From 60ac08413a79636091972aecf286edab34e4a509 Mon Sep 17 00:00:00 2001 From: talha Date: Thu, 26 Oct 2023 20:44:35 +0500 Subject: [PATCH 04/12] Replaced INDEX_NOT_EXIST with IndexNotExist --- packages/components/nodes/vectorstores/Milvus/Milvus_Upsert.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/components/nodes/vectorstores/Milvus/Milvus_Upsert.ts b/packages/components/nodes/vectorstores/Milvus/Milvus_Upsert.ts index ca69cb39..40afe9a4 100644 --- a/packages/components/nodes/vectorstores/Milvus/Milvus_Upsert.ts +++ b/packages/components/nodes/vectorstores/Milvus/Milvus_Upsert.ts @@ -252,7 +252,7 @@ class MilvusUpsert extends Milvus { collection_name: this.collectionName }) - if (descIndexResp.status.error_code === ErrorCode.INDEX_NOT_EXIST) { + if (descIndexResp.status.error_code === ErrorCode.IndexNotExist) { const resp = await this.client.createIndex({ collection_name: this.collectionName, field_name: this.vectorField, From 7aa2e1516e5729cd1dd8ebbc1745198df5671203 Mon Sep 17 00:00:00 2001 From: Ziyi Chen Date: Fri, 27 Oct 2023 19:57:01 -0400 Subject: [PATCH 05/12] should validate .pageContent --- .../nodes/vectorstores/InMemory/InMemoryVectorStore.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/components/nodes/vectorstores/InMemory/InMemoryVectorStore.ts b/packages/components/nodes/vectorstores/InMemory/InMemoryVectorStore.ts index fd2246c9..a480ace0 100644 --- a/packages/components/nodes/vectorstores/InMemory/InMemoryVectorStore.ts +++ b/packages/components/nodes/vectorstores/InMemory/InMemoryVectorStore.ts @@ -71,7 +71,7 @@ class InMemoryVectorStore_VectorStores implements INode { const flattenDocs = docs && docs.length ? flatten(docs) : [] const finalDocs = [] for (let i = 0; i < flattenDocs.length; i += 1) { - if (flattenDocs[i] !== undefined) { + if (flattenDocs[i] !== undefined && flattenDocs[i].pageContent !== undefined) { finalDocs.push(new Document(flattenDocs[i])) } } From 206394bd80ed1ae4ce6ccd50b6022915f2d96e39 Mon Sep 17 00:00:00 2001 From: tirongi Date: Mon, 30 Oct 2023 17:33:27 +0100 Subject: [PATCH 06/12] Add OpenAIEmbeddingCustom This feature allow to select a customised modelName or the default one. --- .../OpenAIEmbeddingCustom.ts | 94 ++++++++++++++++++ .../OpenAIEmbeddingCustom/openai.png | Bin 0 -> 3991 bytes 2 files changed, 94 insertions(+) create mode 100644 packages/components/nodes/embeddings/OpenAIEmbeddingCustom/OpenAIEmbeddingCustom.ts create mode 100644 packages/components/nodes/embeddings/OpenAIEmbeddingCustom/openai.png diff --git a/packages/components/nodes/embeddings/OpenAIEmbeddingCustom/OpenAIEmbeddingCustom.ts b/packages/components/nodes/embeddings/OpenAIEmbeddingCustom/OpenAIEmbeddingCustom.ts new file mode 100644 index 00000000..185236b1 --- /dev/null +++ b/packages/components/nodes/embeddings/OpenAIEmbeddingCustom/OpenAIEmbeddingCustom.ts @@ -0,0 +1,94 @@ +import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface' +import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils' +import { OpenAIEmbeddings, OpenAIEmbeddingsParams } from 'langchain/embeddings/openai' + +class OpenAIEmbeddingCustom_Embeddings implements INode { + label: string + name: string + version: number + type: string + icon: string + category: string + description: string + baseClasses: string[] + credential: INodeParams + inputs: INodeParams[] + + constructor() { + this.label = 'OpenAI Embeddings Custom' + this.name = 'openAIEmbeddingsCustom' + this.version = 1.0 + this.type = 'OpenAIEmbeddingsCustom' + this.icon = 'openai.png' + this.category = 'Embeddings' + this.description = 'OpenAI API to generate embeddings for a given text' + this.baseClasses = [this.type, ...getBaseClasses(OpenAIEmbeddings)] + this.credential = { + label: 'Connect Credential', + name: 'credential', + type: 'credential', + credentialNames: ['openAIApi'] + } + this.inputs = [ + { + label: 'Strip New Lines', + name: 'stripNewLines', + type: 'boolean', + optional: true, + additionalParams: true + }, + { + label: 'Batch Size', + name: 'batchSize', + type: 'number', + optional: true, + additionalParams: true + }, + { + label: 'Timeout', + name: 'timeout', + type: 'number', + optional: true, + additionalParams: true + }, + { + label: 'BasePath', + name: 'basepath', + type: 'string', + optional: true, + additionalParams: true + }, + { + label: 'Model Name', + name: 'modelName', + type: 'string', + optional: true + } + ] + } + + async init(nodeData: INodeData, _: string, options: ICommonObject): Promise { + const stripNewLines = nodeData.inputs?.stripNewLines as boolean + const batchSize = nodeData.inputs?.batchSize as string + const timeout = nodeData.inputs?.timeout as string + const basePath = nodeData.inputs?.basepath as string + const modelName = nodeData.inputs?.modelName as string + + const credentialData = await getCredentialData(nodeData.credential ?? '', options) + const openAIApiKey = getCredentialParam('openAIApiKey', credentialData, nodeData) + + const obj: Partial & { openAIApiKey?: string } = { + openAIApiKey + } + + if (stripNewLines) obj.stripNewLines = stripNewLines + if (batchSize) obj.batchSize = parseInt(batchSize, 10) + if (timeout) obj.timeout = parseInt(timeout, 10) + if (modelName) obj.modelName = modelName + + const model = new OpenAIEmbeddings(obj, { basePath }) + return model + } +} + +module.exports = { nodeClass: OpenAIEmbeddingCustom_Embeddings } diff --git a/packages/components/nodes/embeddings/OpenAIEmbeddingCustom/openai.png b/packages/components/nodes/embeddings/OpenAIEmbeddingCustom/openai.png new file mode 100644 index 0000000000000000000000000000000000000000..de08a05b28979826c4cc669c4899789763a938a1 GIT binary patch literal 3991 zcmV;I4`}d-P)gjhf~eq#s7M4i1UD8XqJW~vA_xftNZ1L8K*&N!(&?H%y1G)Gbam|=aK3jA=g{eX z@7=H7a^Jo8-Gcvf2q9|6MLi;kA{=m2P6cQ2)V1)TAs~(pT*(!*p&9W+1Lr8@H};dw zHug~X$0Z<~j`Sm$E?i7hfWKF8n(eG&Ik{BUEe-a=MOQM|iyKj+xXEW0-3hDfF58I& z#*>GLh)0tE?>F`_krs8`ZF?Zli!3TN1+P64R?{bBi?U;gWH|YTh4+>Hq!L-zB3MBb zV>qpA;HyoBGo%q+*J7AOBx5KtExwO}V$uTc8RtC&hEr%s{OVDVdLga_y~wvLzK??a z^sQ@gj3R+7Tg3NKu$q>k>9{@Whl|Gh`>Pxu-Wg^SlVy}NwlLW4Twghj6#mHhirCmm~&>D3b&!V;S94?d=RK$ zm*UB~=s*f7bfqyD1^9jm$Joe9zT=R}sBsiYlGd-CA*uv8` zKMGwKhufy*PekMhFLJ3|cWa(uw}INL_=N{(5K8gm_}Vt%i};Y<^0aKgz5Hn6RB@I? zbPgQ>S5aV#@Rh7(5HV7%5!}cUN=(wUoD6&QPY6>=!9usII*OSN;4%;Yvb;%^wNdhi0 zr39VgO}gQd>S)X({7RL@S+7<~8R;YeZP;h9LuD+dzpT*K<95F0oFmWPS2oesE^#A? zCxJw|a3xI*65v6kimg0EL#Z|wSMxTf9Ti?g#7&yINO})Ljp#&oy3m&9G$4{NGMHkB zJb^m0!|Z!DK@i3u7IE0@&m-x^1lDq*hLgi9zTOc~NG8|Iib*^p*`&j1 zVploKP_x`!!)y)&T%0EBCZL>e_$&3LI-|ISFDMO}@ZR#C8C!Ep(m9}7r9J{Y#1yf(U)Cd3yJ5ZTF_yw7nmxtCOht;i^v5v`AaL5#Oie195@ zL7;#|%wrb-N14WQ9^!AZwa^&i1CO7YA700^G_+BC^ETRIRx+FQQtXI;h{z7c@EM~? zHZh)}1Di+u324j&5^*AM##oJRe&RKjQw%@^y-8*nKT<^nS#0D^9yJ_OqN4`_PZ%(7 z>DvbLxCDR~b=T`}9)nI~P=LrGCeuOwv<(vto z1WXJ1tvx&=eGlMLUgU^I>-jvZI7)Y55(k5Rzm&U!i(j9`hQ!xPz#dHke&=<%$g{o) zkFi~sdCbj5Mi4LkE{q<$#~Iac@28X20=Q43K_>_(<#TS4BZBI4Cs~HfW2JmfisJVJ zSgo>;Es>AoD!AM53Ec<*0@DLL!A*>mpI{U{SU{n{K8T2%U^Z9CBd8hwBDGL=bYeygf8|aRgNq-z z@iu~PyunFR;){q>@;&#+9)Jk?vY2A&ZyqLT9j4=05h4Q0Si!9UqdXv*ek{|us|PB@ zd`w>=q}pN`!Vgp;lFQ{<6QH392bVqqauozr@r%MJuGW(W`Ne{h?bXV6Z z{&r{`XvjK;2-syh;ITdf_|}4}+}{(S3h(OZ=8Va1I)_r0Fqo& zy~6A2VG=$CA%_x22(SZ{tY$c)*kCd$xE@1UpcXaeBOsdslikxAoYuxb6bT4G5t$4k zoqUt^bZ0Ju11*U@0*>;tsfw#8cTjw|m{)mR+SLy-nSspXw268|+K|DZV2^7kXH9H_ z5!}VvAkmyT7B9mkkV7R|+#wsnjhMk|DoH`3##$LvhGxjWY(W~ijuEg!+STWCjh`88 zn_-1HVANRktSBO$8x6Q1TM$V;B|tIj`4$iD0_a{RSYT;+jb#{3fM~jsLOcg31j^V% z7H4UvW$E>U03;B{sz5F>fOc#)#HN3AZxqRVR?DoCO@amSm0@@uMBHpv7*WFL$s)rF zb1A9n&7~T)3l;G`H^}~_ct+F+eX);#Y5|mHuxjJE{>iXeJu)el_Y5yCB8Qo(u(-4( zU2#6JN0LVL7MI||o5g<~@ItEG>ph<@M8Z>H5;2sK0QBc#` z*M!GeOmm9_1ov0wO3!k#!JcLYbCiW~kD)o`U-FnJ2SE!YSiA?UMZkWkEu#eF(k@uD z0?7up#M+CDG7R1tT51rm&m;jw+~#c{u;L@Kiu+l}SyP=3<67o0U*YLJ{}AJ|6sv1~ z*^J^bw&KCek)}R(vWRJ1ZaLb-nM*GMiQdN(O!Y11Z3Z%#gC;xCl*jmlCoC?5PNA81 z8PwAK^GIh9nI>(90+%s`4;Uyb%;yiJs4?xsPZb*&#c;k+J3?q6g1)@P8wSndJ?MuE z5FESr744NhH~|-vhzls?Q-&~(Y|L3`8!&_qd0r9Z6b$W2XEC>!XvYY2MUADA!!ruC zu_G^W)YR7KyD##vZr4}_0?WBm#oRmzyVGg?7}~Bv~CU<`e#`@VgFXorz1$zH*YebeC)Mwbq?C{es?{ zCg5Ey9W}9r4t9*0ia47VJgG4_gG~mJb$<7|+cL3M`X#3cn5Z=YM)~>Wynfdl#jY;U znOFJE1O*|#*1Q4c9YH(zhwl4B(;OvWvDK(CB0O^{~(@5xY5g*h@jj>*H7jcq+Y%QmGGz)Z9q$hN_ zf;9@`d8F>t7%w?SfQRR_qsCV1t}a;UvWK0Fm92sTNxaN)o%MPN(7K&&hJc+~3m`P& zM?*s@aOmCZBf=`GoXjf-)XBYs3!7yk^;GpBY2>%at5u-8(9;q>MP)7PEdY~(e* z0I(?myTE>)W1+sQvtDFVV$qOkR{XuZ!vZb&D!V6f?^G5?ug(>yjyw|PvxWLQkFqzl%f#=ONpHAVts zCG(iIQtoWFNaZLow*`7JpLxvr%hcIh#i>4)=Ng|Qv#1oA` zIYpcxpKP{sKuUt;!&NNd66{UTmd{;mwXr^vh@t_FXi8HW6R&DN2xFp!kea|#?BAi# z0PI6cR@*lJJ&0tT7rq8V=xdWo?Lj1uUUe;waR{W^^eV2?48IUx#RXA}qu3G!9z=>5 za~_A`Yap6&7Dj>h>5sWE-$my`BqI$c?W!($47+fjz7GO@SZ!ictR#zG7v|irjTTIh z{Qi1h%9_Xc3vc5K1{d9!NuI9P^5&62SEtmTx*SsBbfiDYT%r16=2L9vYgUkp+o?{} z{hX@#YHpEo3i*wF>|h&volfvm_XK!x-oBju50C!=Nj{KH?md;N0000bbVXQnWMOn= zI%9HWVRU5xGB7eSEigGPF*Z~%IXW;nIx#mZFfckWFtzvO1ONa4C3HntbYx+4Wjbwd xWNBu305UK#GA%GUEipD!FgZFfI65&mD=;uRFfhcbT(|%L002ovPDHLkV1j%(J<0$8 literal 0 HcmV?d00001 From 7bc41939e4ad99c6f1daef02755df9a6a3b12bf2 Mon Sep 17 00:00:00 2001 From: SebastjanPrachovskij Date: Tue, 31 Oct 2023 16:27:37 +0200 Subject: [PATCH 07/12] Add SearchApi documentloader, tool & podcast QA example --- .../credentials/SearchApi.credential.ts | 24 + .../documentloaders/SearchApi/SearchAPI.ts | 109 +++ .../documentloaders/SearchApi/searchapi.svg | 1 + .../nodes/tools/SearchApi/SearchAPI.ts | 42 ++ .../nodes/tools/SearchApi/searchapi.svg | 1 + .../chatflows/Chat with a Podcast.json | 702 ++++++++++++++++++ 6 files changed, 879 insertions(+) create mode 100644 packages/components/credentials/SearchApi.credential.ts create mode 100644 packages/components/nodes/documentloaders/SearchApi/SearchAPI.ts create mode 100644 packages/components/nodes/documentloaders/SearchApi/searchapi.svg create mode 100644 packages/components/nodes/tools/SearchApi/SearchAPI.ts create mode 100644 packages/components/nodes/tools/SearchApi/searchapi.svg create mode 100644 packages/server/marketplaces/chatflows/Chat with a Podcast.json diff --git a/packages/components/credentials/SearchApi.credential.ts b/packages/components/credentials/SearchApi.credential.ts new file mode 100644 index 00000000..ccc65a6c --- /dev/null +++ b/packages/components/credentials/SearchApi.credential.ts @@ -0,0 +1,24 @@ +import { INodeParams, INodeCredential } from '../src/Interface' + +class SearchApi implements INodeCredential { + label: string + name: string + version: number + description: string + inputs: INodeParams[] + + constructor() { + this.label = 'Search API' + this.name = 'searchApi' + this.version = 1.0 + this.inputs = [ + { + label: 'SearchApi API Key', + name: 'searchApiKey', + type: 'password' + } + ] + } +} + +module.exports = { credClass: SearchApi } diff --git a/packages/components/nodes/documentloaders/SearchApi/SearchAPI.ts b/packages/components/nodes/documentloaders/SearchApi/SearchAPI.ts new file mode 100644 index 00000000..9d14dc5a --- /dev/null +++ b/packages/components/nodes/documentloaders/SearchApi/SearchAPI.ts @@ -0,0 +1,109 @@ +import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface' +import { TextSplitter } from 'langchain/text_splitter' +import { SearchApiLoader } from 'langchain/document_loaders/web/searchapi' +import { getCredentialData, getCredentialParam } from '../../../src' + +// Provides access to multiple search engines using the SearchApi. +// For available parameters & engines, refer to: https://www.searchapi.io/docs/google +class SearchAPI_DocumentLoaders implements INode { + label: string + name: string + version: number + description: string + type: string + icon: string + category: string + baseClasses: string[] + credential: INodeParams + inputs: INodeParams[] + + constructor() { + this.label = 'SearchApi For Web Search' + this.name = 'searchApi' + this.version = 1.0 + this.type = 'Document' + this.icon = 'searchapi.svg' + this.category = 'Document Loaders' + this.description = 'Load data from real-time search results' + this.baseClasses = [this.type] + this.credential = { + label: 'Connect Credential', + name: 'credential', + type: 'credential', + optional: false, + credentialNames: ['searchApi'] + } + this.inputs = [ + { + label: 'Query', + name: 'query', + type: 'string', + optional: true + }, + { + label: 'Custom Parameters', + name: 'customParameters', + type: 'json', + optional: true, + additionalParams: true + }, + { + label: 'Text Splitter', + name: 'textSplitter', + type: 'TextSplitter', + optional: true + }, + { + label: 'Metadata', + name: 'metadata', + type: 'json', + optional: true, + additionalParams: true + } + ] + } + + async init(nodeData: INodeData, _: string, options: ICommonObject): Promise { + const textSplitter = nodeData.inputs?.textSplitter as TextSplitter + const query = nodeData.inputs?.query as string + const customParameters = nodeData.inputs?.customParameters + const metadata = nodeData.inputs?.metadata + + // Fetch the API credentials for this node + const credentialData = await getCredentialData(nodeData.credential ?? '', options) + const searchApiKey = getCredentialParam('searchApiKey', credentialData, nodeData) + + // Check and parse custom parameters (should be JSON or object) + const parsedParameters = typeof customParameters === 'object' ? customParameters : JSON.parse(customParameters || '{}'); + + // Prepare the configuration for the SearchApiLoader + const loaderConfig = { + q: query, + apiKey: searchApiKey, + ...parsedParameters + }; + + // Initialize the loader with the given configuration + const loader = new SearchApiLoader(loaderConfig); + + // Fetch documents, split if a text splitter is provided + const docs = textSplitter ? await loader.loadAndSplit() : await loader.load() + + if (metadata) { + const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) + return docs.map((doc) => { + return { + ...doc, + metadata: { + ...doc.metadata, + ...parsedMetadata + } + } + }) + } + + return docs + } +} + +module.exports = { nodeClass: SearchAPI_DocumentLoaders } diff --git a/packages/components/nodes/documentloaders/SearchApi/searchapi.svg b/packages/components/nodes/documentloaders/SearchApi/searchapi.svg new file mode 100644 index 00000000..c44c29c4 --- /dev/null +++ b/packages/components/nodes/documentloaders/SearchApi/searchapi.svg @@ -0,0 +1 @@ +0479_octopus_verti diff --git a/packages/components/nodes/tools/SearchApi/SearchAPI.ts b/packages/components/nodes/tools/SearchApi/SearchAPI.ts new file mode 100644 index 00000000..3f760087 --- /dev/null +++ b/packages/components/nodes/tools/SearchApi/SearchAPI.ts @@ -0,0 +1,42 @@ +import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface' +import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils' +import { SearchApi } from 'langchain/tools' + +class SearchAPI_Tools implements INode { + label: string + name: string + version: number + description: string + type: string + icon: string + category: string + baseClasses: string[] + credential: INodeParams + inputs: INodeParams[] + + constructor() { + this.label = 'SearchApi' + this.name = 'searchAPI' + this.version = 1.0 + this.type = 'SearchAPI' + this.icon = 'searchapi.svg' + this.category = 'Tools' + this.description = 'Real-time API for accessing Google Search data' + this.inputs = [] + this.credential = { + label: 'Connect Credential', + name: 'credential', + type: 'credential', + credentialNames: ['searchApi'] + } + this.baseClasses = [this.type, ...getBaseClasses(SearchApi)] + } + + async init(nodeData: INodeData, _: string, options: ICommonObject): Promise { + const credentialData = await getCredentialData(nodeData.credential ?? '', options) + const searchApiKey = getCredentialParam('searchApiKey', credentialData, nodeData) + return new SearchApi(searchApiKey) + } +} + +module.exports = { nodeClass: SearchAPI_Tools } diff --git a/packages/components/nodes/tools/SearchApi/searchapi.svg b/packages/components/nodes/tools/SearchApi/searchapi.svg new file mode 100644 index 00000000..c44c29c4 --- /dev/null +++ b/packages/components/nodes/tools/SearchApi/searchapi.svg @@ -0,0 +1 @@ +0479_octopus_verti diff --git a/packages/server/marketplaces/chatflows/Chat with a Podcast.json b/packages/server/marketplaces/chatflows/Chat with a Podcast.json new file mode 100644 index 00000000..e91bf50f --- /dev/null +++ b/packages/server/marketplaces/chatflows/Chat with a Podcast.json @@ -0,0 +1,702 @@ +{ + "description": "Engage with data sources such as YouTube Transcripts, Google, and more through intelligent Q&A interactions", + "nodes": [ + { + "width": 300, + "height": 483, + "id": "conversationalRetrievalQAChain_0", + "position": { + "x": 1499.2693059023254, + "y": 430.03911199833317 + }, + "type": "customNode", + "data": { + "id": "conversationalRetrievalQAChain_0", + "label": "Conversational Retrieval QA Chain", + "version": 1, + "name": "conversationalRetrievalQAChain", + "type": "ConversationalRetrievalQAChain", + "baseClasses": [ + "ConversationalRetrievalQAChain", + "BaseChain", + "Runnable" + ], + "category": "Chains", + "description": "Document QA - built on RetrievalQAChain to provide a chat history component", + "inputParams": [ + { + "label": "Return Source Documents", + "name": "returnSourceDocuments", + "type": "boolean", + "optional": true, + "id": "conversationalRetrievalQAChain_0-input-returnSourceDocuments-boolean" + }, + { + "label": "System Message", + "name": "systemMessagePrompt", + "type": "string", + "rows": 4, + "additionalParams": true, + "optional": true, + "placeholder": "I want you to act as a document that I am having a conversation with. Your name is \"AI Assistant\". You will provide me with answers from the given info. If the answer is not included, say exactly \"Hmm, I am not sure.\" and stop after that. Refuse to answer any question not about the info. Never break character.", + "id": "conversationalRetrievalQAChain_0-input-systemMessagePrompt-string" + }, + { + "label": "Chain Option", + "name": "chainOption", + "type": "options", + "options": [ + { + "label": "MapReduceDocumentsChain", + "name": "map_reduce", + "description": "Suitable for QA tasks over larger documents and can run the preprocessing step in parallel, reducing the running time" + }, + { + "label": "RefineDocumentsChain", + "name": "refine", + "description": "Suitable for QA tasks over a large number of documents." + }, + { + "label": "StuffDocumentsChain", + "name": "stuff", + "description": "Suitable for QA tasks over a small number of documents." + } + ], + "additionalParams": true, + "optional": true, + "id": "conversationalRetrievalQAChain_0-input-chainOption-options" + } + ], + "inputAnchors": [ + { + "label": "Language Model", + "name": "model", + "type": "BaseLanguageModel", + "id": "conversationalRetrievalQAChain_0-input-model-BaseLanguageModel" + }, + { + "label": "Vector Store Retriever", + "name": "vectorStoreRetriever", + "type": "BaseRetriever", + "id": "conversationalRetrievalQAChain_0-input-vectorStoreRetriever-BaseRetriever" + }, + { + "label": "Memory", + "name": "memory", + "type": "BaseMemory", + "optional": true, + "description": "If left empty, a default BufferMemory will be used", + "id": "conversationalRetrievalQAChain_0-input-memory-BaseMemory" + } + ], + "inputs": { + "model": "{{chatOpenAI_0.data.instance}}", + "vectorStoreRetriever": "{{memoryVectorStore_0.data.instance}}", + "memory": "", + "returnSourceDocuments": "", + "systemMessagePrompt": "", + "chainOption": "" + }, + "outputAnchors": [ + { + "id": "conversationalRetrievalQAChain_0-output-conversationalRetrievalQAChain-ConversationalRetrievalQAChain|BaseChain|Runnable", + "name": "conversationalRetrievalQAChain", + "label": "ConversationalRetrievalQAChain", + "type": "ConversationalRetrievalQAChain | BaseChain | Runnable" + } + ], + "outputs": {}, + "selected": false + }, + "selected": false, + "positionAbsolute": { + "x": 1499.2693059023254, + "y": 430.03911199833317 + }, + "dragging": false + }, + { + "width": 300, + "height": 408, + "id": "memoryVectorStore_0", + "position": { + "x": 1082.0280622332507, + "y": 589.9990964387842 + }, + "type": "customNode", + "data": { + "id": "memoryVectorStore_0", + "label": "In-Memory Vector Store", + "version": 1, + "name": "memoryVectorStore", + "type": "Memory", + "baseClasses": [ + "Memory", + "VectorStoreRetriever", + "BaseRetriever" + ], + "category": "Vector Stores", + "description": "In-memory vectorstore that stores embeddings and does an exact, linear search for the most similar embeddings.", + "inputParams": [ + { + "label": "Top K", + "name": "topK", + "description": "Number of top results to fetch. Default to 4", + "placeholder": "4", + "type": "number", + "optional": true, + "id": "memoryVectorStore_0-input-topK-number" + } + ], + "inputAnchors": [ + { + "label": "Document", + "name": "document", + "type": "Document", + "list": true, + "id": "memoryVectorStore_0-input-document-Document" + }, + { + "label": "Embeddings", + "name": "embeddings", + "type": "Embeddings", + "id": "memoryVectorStore_0-input-embeddings-Embeddings" + } + ], + "inputs": { + "document": [ + "{{searchApi_0.data.instance}}", + "{{searchApi_0.data.instance}}", + "{{searchApi_0.data.instance}}" + ], + "embeddings": "{{openAIEmbeddings_0.data.instance}}", + "topK": "" + }, + "outputAnchors": [ + { + "name": "output", + "label": "Output", + "type": "options", + "options": [ + { + "id": "memoryVectorStore_0-output-retriever-Memory|VectorStoreRetriever|BaseRetriever", + "name": "retriever", + "label": "Memory Retriever", + "type": "Memory | VectorStoreRetriever | BaseRetriever" + }, + { + "id": "memoryVectorStore_0-output-vectorStore-Memory|VectorStore", + "name": "vectorStore", + "label": "Memory Vector Store", + "type": "Memory | VectorStore" + } + ], + "default": "retriever" + } + ], + "outputs": { + "output": "retriever" + }, + "selected": false + }, + "positionAbsolute": { + "x": 1082.0280622332507, + "y": 589.9990964387842 + }, + "selected": false, + "dragging": false + }, + { + "width": 300, + "height": 577, + "id": "chatOpenAI_0", + "position": { + "x": 1056.2788608917747, + "y": -60.59149112477064 + }, + "type": "customNode", + "data": { + "id": "chatOpenAI_0", + "label": "ChatOpenAI", + "version": 2, + "name": "chatOpenAI", + "type": "ChatOpenAI", + "baseClasses": [ + "ChatOpenAI", + "BaseChatModel", + "BaseLanguageModel", + "Runnable" + ], + "category": "Chat Models", + "description": "Wrapper around OpenAI large language models that use the Chat endpoint", + "inputParams": [ + { + "label": "Connect Credential", + "name": "credential", + "type": "credential", + "credentialNames": [ + "openAIApi" + ], + "id": "chatOpenAI_0-input-credential-credential" + }, + { + "label": "Model Name", + "name": "modelName", + "type": "options", + "options": [ + { + "label": "gpt-4", + "name": "gpt-4" + }, + { + "label": "gpt-4-0613", + "name": "gpt-4-0613" + }, + { + "label": "gpt-4-32k", + "name": "gpt-4-32k" + }, + { + "label": "gpt-4-32k-0613", + "name": "gpt-4-32k-0613" + }, + { + "label": "gpt-3.5-turbo", + "name": "gpt-3.5-turbo" + }, + { + "label": "gpt-3.5-turbo-0613", + "name": "gpt-3.5-turbo-0613" + }, + { + "label": "gpt-3.5-turbo-16k", + "name": "gpt-3.5-turbo-16k" + }, + { + "label": "gpt-3.5-turbo-16k-0613", + "name": "gpt-3.5-turbo-16k-0613" + } + ], + "default": "gpt-3.5-turbo", + "optional": true, + "id": "chatOpenAI_0-input-modelName-options" + }, + { + "label": "Temperature", + "name": "temperature", + "type": "number", + "step": 0.1, + "default": 0.9, + "optional": true, + "id": "chatOpenAI_0-input-temperature-number" + }, + { + "label": "Max Tokens", + "name": "maxTokens", + "type": "number", + "step": 1, + "optional": true, + "additionalParams": true, + "id": "chatOpenAI_0-input-maxTokens-number" + }, + { + "label": "Top Probability", + "name": "topP", + "type": "number", + "step": 0.1, + "optional": true, + "additionalParams": true, + "id": "chatOpenAI_0-input-topP-number" + }, + { + "label": "Frequency Penalty", + "name": "frequencyPenalty", + "type": "number", + "step": 0.1, + "optional": true, + "additionalParams": true, + "id": "chatOpenAI_0-input-frequencyPenalty-number" + }, + { + "label": "Presence Penalty", + "name": "presencePenalty", + "type": "number", + "step": 0.1, + "optional": true, + "additionalParams": true, + "id": "chatOpenAI_0-input-presencePenalty-number" + }, + { + "label": "Timeout", + "name": "timeout", + "type": "number", + "step": 1, + "optional": true, + "additionalParams": true, + "id": "chatOpenAI_0-input-timeout-number" + }, + { + "label": "BasePath", + "name": "basepath", + "type": "string", + "optional": true, + "additionalParams": true, + "id": "chatOpenAI_0-input-basepath-string" + }, + { + "label": "BaseOptions", + "name": "baseOptions", + "type": "json", + "optional": true, + "additionalParams": true, + "id": "chatOpenAI_0-input-baseOptions-json" + } + ], + "inputAnchors": [ + { + "label": "Cache", + "name": "cache", + "type": "BaseCache", + "optional": true, + "id": "chatOpenAI_0-input-cache-BaseCache" + } + ], + "inputs": { + "cache": "", + "modelName": "gpt-3.5-turbo", + "temperature": "0.5", + "maxTokens": "", + "topP": "", + "frequencyPenalty": "", + "presencePenalty": "", + "timeout": "", + "basepath": "", + "baseOptions": "" + }, + "outputAnchors": [ + { + "id": "chatOpenAI_0-output-chatOpenAI-ChatOpenAI|BaseChatModel|BaseLanguageModel|Runnable", + "name": "chatOpenAI", + "label": "ChatOpenAI", + "type": "ChatOpenAI | BaseChatModel | BaseLanguageModel | Runnable" + } + ], + "outputs": {}, + "selected": false + }, + "selected": false, + "positionAbsolute": { + "x": 1056.2788608917747, + "y": -60.59149112477064 + }, + "dragging": false + }, + { + "width": 300, + "height": 478, + "id": "characterTextSplitter_0", + "position": { + "x": 260.5475803279806, + "y": -65.1647664861618 + }, + "type": "customNode", + "data": { + "id": "characterTextSplitter_0", + "label": "Character Text Splitter", + "version": 1, + "name": "characterTextSplitter", + "type": "CharacterTextSplitter", + "baseClasses": [ + "CharacterTextSplitter", + "TextSplitter", + "BaseDocumentTransformer", + "Runnable" + ], + "category": "Text Splitters", + "description": "splits only on one type of character (defaults to \"\\n\\n\").", + "inputParams": [ + { + "label": "Chunk Size", + "name": "chunkSize", + "type": "number", + "default": 1000, + "optional": true, + "id": "characterTextSplitter_0-input-chunkSize-number" + }, + { + "label": "Chunk Overlap", + "name": "chunkOverlap", + "type": "number", + "optional": true, + "id": "characterTextSplitter_0-input-chunkOverlap-number" + }, + { + "label": "Custom Separator", + "name": "separator", + "type": "string", + "placeholder": "\" \"", + "description": "Seperator to determine when to split the text, will override the default separator", + "optional": true, + "id": "characterTextSplitter_0-input-separator-string" + } + ], + "inputAnchors": [], + "inputs": { + "chunkSize": "2000", + "chunkOverlap": "200", + "separator": "" + }, + "outputAnchors": [ + { + "id": "characterTextSplitter_0-output-characterTextSplitter-CharacterTextSplitter|TextSplitter|BaseDocumentTransformer|Runnable", + "name": "characterTextSplitter", + "label": "CharacterTextSplitter", + "type": "CharacterTextSplitter | TextSplitter | BaseDocumentTransformer | Runnable" + } + ], + "outputs": {}, + "selected": false + }, + "selected": false, + "positionAbsolute": { + "x": 260.5475803279806, + "y": -65.1647664861618 + }, + "dragging": false + }, + { + "width": 300, + "height": 332, + "id": "openAIEmbeddings_0", + "position": { + "x": 666.3950526535211, + "y": 777.4191705193945 + }, + "type": "customNode", + "data": { + "id": "openAIEmbeddings_0", + "label": "OpenAI Embeddings", + "version": 1, + "name": "openAIEmbeddings", + "type": "OpenAIEmbeddings", + "baseClasses": [ + "OpenAIEmbeddings", + "Embeddings" + ], + "category": "Embeddings", + "description": "OpenAI API to generate embeddings for a given text", + "inputParams": [ + { + "label": "Connect Credential", + "name": "credential", + "type": "credential", + "credentialNames": [ + "openAIApi" + ], + "id": "openAIEmbeddings_0-input-credential-credential" + }, + { + "label": "Strip New Lines", + "name": "stripNewLines", + "type": "boolean", + "optional": true, + "additionalParams": true, + "id": "openAIEmbeddings_0-input-stripNewLines-boolean" + }, + { + "label": "Batch Size", + "name": "batchSize", + "type": "number", + "optional": true, + "additionalParams": true, + "id": "openAIEmbeddings_0-input-batchSize-number" + }, + { + "label": "Timeout", + "name": "timeout", + "type": "number", + "optional": true, + "additionalParams": true, + "id": "openAIEmbeddings_0-input-timeout-number" + }, + { + "label": "BasePath", + "name": "basepath", + "type": "string", + "optional": true, + "additionalParams": true, + "id": "openAIEmbeddings_0-input-basepath-string" + } + ], + "inputAnchors": [], + "inputs": { + "stripNewLines": "", + "batchSize": "", + "timeout": "", + "basepath": "" + }, + "outputAnchors": [ + { + "id": "openAIEmbeddings_0-output-openAIEmbeddings-OpenAIEmbeddings|Embeddings", + "name": "openAIEmbeddings", + "label": "OpenAIEmbeddings", + "type": "OpenAIEmbeddings | Embeddings" + } + ], + "outputs": {}, + "selected": false + }, + "selected": false, + "dragging": false, + "positionAbsolute": { + "x": 666.3950526535211, + "y": 777.4191705193945 + } + }, + { + "width": 300, + "height": 482, + "id": "searchApi_0", + "position": { + "x": 680.1258121447145, + "y": 144.9905217023999 + }, + "type": "customNode", + "data": { + "id": "searchApi_0", + "label": "SearchApi", + "version": 1, + "name": "searchApi", + "type": "Document", + "baseClasses": [ + "Document" + ], + "category": "Document Loaders", + "description": "Load data from real-time search results", + "inputParams": [ + { + "label": "Connect Credential", + "name": "credential", + "type": "credential", + "optional": false, + "credentialNames": [ + "searchApi" + ], + "id": "searchApi_0-input-credential-credential" + }, + { + "label": "Query", + "name": "query", + "type": "string", + "optional": true, + "id": "searchApi_0-input-query-string" + }, + { + "label": "Custom Parameters", + "name": "customParameters", + "type": "json", + "optional": true, + "additionalParams": true, + "id": "searchApi_0-input-customParameters-json" + }, + { + "label": "Metadata", + "name": "metadata", + "type": "json", + "optional": true, + "additionalParams": true, + "id": "searchApi_0-input-metadata-json" + } + ], + "inputAnchors": [ + { + "label": "Text Splitter", + "name": "textSplitter", + "type": "TextSplitter", + "optional": true, + "id": "searchApi_0-input-textSplitter-TextSplitter" + } + ], + "inputs": { + "query": "", + "customParameters": "{\"engine\":\"youtube_transcripts\",\"video_id\":\"0e3GPea1Tyg\"}", + "textSplitter": "{{characterTextSplitter_0.data.instance}}", + "metadata": "" + }, + "outputAnchors": [ + { + "id": "searchApi_0-output-searchApi-Document", + "name": "searchApi", + "label": "Document", + "type": "Document" + } + ], + "outputs": {}, + "selected": false + }, + "selected": false, + "positionAbsolute": { + "x": 680.1258121447145, + "y": 144.9905217023999 + }, + "dragging": false + } + ], + "edges": [ + { + "source": "memoryVectorStore_0", + "sourceHandle": "memoryVectorStore_0-output-retriever-Memory|VectorStoreRetriever|BaseRetriever", + "target": "conversationalRetrievalQAChain_0", + "targetHandle": "conversationalRetrievalQAChain_0-input-vectorStoreRetriever-BaseRetriever", + "type": "buttonedge", + "id": "memoryVectorStore_0-memoryVectorStore_0-output-retriever-Memory|VectorStoreRetriever|BaseRetriever-conversationalRetrievalQAChain_0-conversationalRetrievalQAChain_0-input-vectorStoreRetriever-BaseRetriever", + "data": { + "label": "" + } + }, + { + "source": "chatOpenAI_0", + "sourceHandle": "chatOpenAI_0-output-chatOpenAI-ChatOpenAI|BaseChatModel|BaseLanguageModel|Runnable", + "target": "conversationalRetrievalQAChain_0", + "targetHandle": "conversationalRetrievalQAChain_0-input-model-BaseLanguageModel", + "type": "buttonedge", + "id": "chatOpenAI_0-chatOpenAI_0-output-chatOpenAI-ChatOpenAI|BaseChatModel|BaseLanguageModel|Runnable-conversationalRetrievalQAChain_0-conversationalRetrievalQAChain_0-input-model-BaseLanguageModel", + "data": { + "label": "" + } + }, + { + "source": "openAIEmbeddings_0", + "sourceHandle": "openAIEmbeddings_0-output-openAIEmbeddings-OpenAIEmbeddings|Embeddings", + "target": "memoryVectorStore_0", + "targetHandle": "memoryVectorStore_0-input-embeddings-Embeddings", + "type": "buttonedge", + "id": "openAIEmbeddings_0-openAIEmbeddings_0-output-openAIEmbeddings-OpenAIEmbeddings|Embeddings-memoryVectorStore_0-memoryVectorStore_0-input-embeddings-Embeddings", + "data": { + "label": "" + } + }, + { + "source": "characterTextSplitter_0", + "sourceHandle": "characterTextSplitter_0-output-characterTextSplitter-CharacterTextSplitter|TextSplitter|BaseDocumentTransformer|Runnable", + "target": "searchApi_0", + "targetHandle": "searchApi_0-input-textSplitter-TextSplitter", + "type": "buttonedge", + "id": "characterTextSplitter_0-characterTextSplitter_0-output-characterTextSplitter-CharacterTextSplitter|TextSplitter|BaseDocumentTransformer|Runnable-searchApi_0-searchApi_0-input-textSplitter-TextSplitter", + "data": { + "label": "" + } + }, + { + "source": "searchApi_0", + "sourceHandle": "searchApi_0-output-searchApi-Document", + "target": "memoryVectorStore_0", + "targetHandle": "memoryVectorStore_0-input-document-Document", + "type": "buttonedge", + "id": "searchApi_0-searchApi_0-output-searchApi-Document-memoryVectorStore_0-memoryVectorStore_0-input-document-Document", + "data": { + "label": "" + } + } + ] +} From 039d8d26be95d19ba014d4e716820b6e984bc3e2 Mon Sep 17 00:00:00 2001 From: Henry Date: Tue, 31 Oct 2023 16:32:33 +0000 Subject: [PATCH 08/12] expand text file supported file types --- .../nodes/documentloaders/Text/Text.ts | 5 +++-- .../RecursiveCharacterTextSplitter.ts | 1 + .../Conversational Retrieval QA Chain.json | 17 ++++++++++++++--- .../marketplaces/chatflows/Local QnA.json | 17 ++++++++++++++--- .../chatflows/Metadata Filter Upsert.json | 17 ++++++++++++++--- 5 files changed, 46 insertions(+), 11 deletions(-) diff --git a/packages/components/nodes/documentloaders/Text/Text.ts b/packages/components/nodes/documentloaders/Text/Text.ts index c3e3b61e..3f12e490 100644 --- a/packages/components/nodes/documentloaders/Text/Text.ts +++ b/packages/components/nodes/documentloaders/Text/Text.ts @@ -19,7 +19,7 @@ class Text_DocumentLoaders implements INode { constructor() { this.label = 'Text File' this.name = 'textFile' - this.version = 2.0 + this.version = 3.0 this.type = 'Document' this.icon = 'textFile.svg' this.category = 'Document Loaders' @@ -30,7 +30,8 @@ class Text_DocumentLoaders implements INode { label: 'Txt File', name: 'txtFile', type: 'file', - fileType: '.txt' + fileType: + '.txt, .html, .aspx, .asp, .cpp, .c, .cs, .css, .go, .h, .java, .js, .less, .ts, .php, .proto, .python, .py, .rst, .ruby, .rb, .rs, .scala, .sc, .scss, .sol, .sql, .swift, .markdown, .md, .tex, .ltx, .vb, .xml' }, { label: 'Text Splitter', diff --git a/packages/components/nodes/textsplitters/RecursiveCharacterTextSplitter/RecursiveCharacterTextSplitter.ts b/packages/components/nodes/textsplitters/RecursiveCharacterTextSplitter/RecursiveCharacterTextSplitter.ts index d7dbc1c3..13386bd9 100644 --- a/packages/components/nodes/textsplitters/RecursiveCharacterTextSplitter/RecursiveCharacterTextSplitter.ts +++ b/packages/components/nodes/textsplitters/RecursiveCharacterTextSplitter/RecursiveCharacterTextSplitter.ts @@ -43,6 +43,7 @@ class RecursiveCharacterTextSplitter_TextSplitters implements INode { rows: 4, description: 'Array of custom separators to determine when to split the text, will override the default separators', placeholder: `["|", "##", ">", "-"]`, + additionalParams: true, optional: true } ] diff --git a/packages/server/marketplaces/chatflows/Conversational Retrieval QA Chain.json b/packages/server/marketplaces/chatflows/Conversational Retrieval QA Chain.json index 8bbb904e..38e780cc 100644 --- a/packages/server/marketplaces/chatflows/Conversational Retrieval QA Chain.json +++ b/packages/server/marketplaces/chatflows/Conversational Retrieval QA Chain.json @@ -368,7 +368,7 @@ "id": "recursiveCharacterTextSplitter_0", "label": "Recursive Character Text Splitter", "name": "recursiveCharacterTextSplitter", - "version": 1, + "version": 2, "type": "RecursiveCharacterTextSplitter", "baseClasses": ["RecursiveCharacterTextSplitter", "TextSplitter"], "category": "Text Splitters", @@ -388,6 +388,17 @@ "type": "number", "optional": true, "id": "recursiveCharacterTextSplitter_0-input-chunkOverlap-number" + }, + { + "label": "Custom Separators", + "name": "separators", + "type": "string", + "rows": 4, + "description": "Array of custom separators to determine when to split the text, will override the default separators", + "placeholder": "[\"|\", \"##\", \">\", \"-\"]", + "additionalParams": true, + "optional": true, + "id": "recursiveCharacterTextSplitter_0-input-separators-string" } ], "inputAnchors": [], @@ -426,7 +437,7 @@ "id": "textFile_0", "label": "Text File", "name": "textFile", - "version": 2, + "version": 3, "type": "Document", "baseClasses": ["Document"], "category": "Document Loaders", @@ -436,7 +447,7 @@ "label": "Txt File", "name": "txtFile", "type": "file", - "fileType": ".txt", + "fileType": ".txt, .html, .aspx, .asp, .cpp, .c, .cs, .css, .go, .h, .java, .js, .less, .ts, .php, .proto, .python, .py, .rst, .ruby, .rb, .rs, .scala, .sc, .scss, .sol, .sql, .swift, .markdown, .md, .tex, .ltx, .vb, .xml", "id": "textFile_0-input-txtFile-file" }, { diff --git a/packages/server/marketplaces/chatflows/Local QnA.json b/packages/server/marketplaces/chatflows/Local QnA.json index fb13ba21..78ce16f6 100644 --- a/packages/server/marketplaces/chatflows/Local QnA.json +++ b/packages/server/marketplaces/chatflows/Local QnA.json @@ -14,7 +14,7 @@ "id": "recursiveCharacterTextSplitter_1", "label": "Recursive Character Text Splitter", "name": "recursiveCharacterTextSplitter", - "version": 1, + "version": 2, "type": "RecursiveCharacterTextSplitter", "baseClasses": ["RecursiveCharacterTextSplitter", "TextSplitter"], "category": "Text Splitters", @@ -34,6 +34,17 @@ "type": "number", "optional": true, "id": "recursiveCharacterTextSplitter_1-input-chunkOverlap-number" + }, + { + "label": "Custom Separators", + "name": "separators", + "type": "string", + "rows": 4, + "description": "Array of custom separators to determine when to split the text, will override the default separators", + "placeholder": "[\"|\", \"##\", \">\", \"-\"]", + "additionalParams": true, + "optional": true, + "id": "recursiveCharacterTextSplitter_1-input-separators-string" } ], "inputAnchors": [], @@ -373,7 +384,7 @@ "id": "textFile_0", "label": "Text File", "name": "textFile", - "version": 1, + "version": 3, "type": "Document", "baseClasses": ["Document"], "category": "Document Loaders", @@ -383,7 +394,7 @@ "label": "Txt File", "name": "txtFile", "type": "file", - "fileType": ".txt", + "fileType": ".txt, .html, .aspx, .asp, .cpp, .c, .cs, .css, .go, .h, .java, .js, .less, .ts, .php, .proto, .python, .py, .rst, .ruby, .rb, .rs, .scala, .sc, .scss, .sol, .sql, .swift, .markdown, .md, .tex, .ltx, .vb, .xml", "id": "textFile_0-input-txtFile-file" }, { diff --git a/packages/server/marketplaces/chatflows/Metadata Filter Upsert.json b/packages/server/marketplaces/chatflows/Metadata Filter Upsert.json index 6e79301e..38cec6dd 100644 --- a/packages/server/marketplaces/chatflows/Metadata Filter Upsert.json +++ b/packages/server/marketplaces/chatflows/Metadata Filter Upsert.json @@ -14,7 +14,7 @@ "id": "recursiveCharacterTextSplitter_1", "label": "Recursive Character Text Splitter", "name": "recursiveCharacterTextSplitter", - "version": 1, + "version": 2, "type": "RecursiveCharacterTextSplitter", "baseClasses": ["RecursiveCharacterTextSplitter", "TextSplitter"], "category": "Text Splitters", @@ -34,6 +34,17 @@ "type": "number", "optional": true, "id": "recursiveCharacterTextSplitter_1-input-chunkOverlap-number" + }, + { + "label": "Custom Separators", + "name": "separators", + "type": "string", + "rows": 4, + "description": "Array of custom separators to determine when to split the text, will override the default separators", + "placeholder": "[\"|\", \"##\", \">\", \"-\"]", + "additionalParams": true, + "optional": true, + "id": "recursiveCharacterTextSplitter_1-input-separators-string" } ], "inputAnchors": [], @@ -72,7 +83,7 @@ "id": "textFile_0", "label": "Text File", "name": "textFile", - "version": 1, + "version": 3, "type": "Document", "baseClasses": ["Document"], "category": "Document Loaders", @@ -82,7 +93,7 @@ "label": "Txt File", "name": "txtFile", "type": "file", - "fileType": ".txt", + "fileType": ".txt, .html, .aspx, .asp, .cpp, .c, .cs, .css, .go, .h, .java, .js, .less, .ts, .php, .proto, .python, .py, .rst, .ruby, .rb, .rs, .scala, .sc, .scss, .sol, .sql, .swift, .markdown, .md, .tex, .ltx, .vb, .xml", "id": "textFile_0-input-txtFile-file" }, { From 83947dc55d242938e195945bf4391d94c424f3a0 Mon Sep 17 00:00:00 2001 From: SebastjanPrachovskij Date: Wed, 1 Nov 2023 18:19:05 +0200 Subject: [PATCH 09/12] Add SearchApi credentials description --- packages/components/credentials/SearchApi.credential.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/components/credentials/SearchApi.credential.ts b/packages/components/credentials/SearchApi.credential.ts index ccc65a6c..96d3dc01 100644 --- a/packages/components/credentials/SearchApi.credential.ts +++ b/packages/components/credentials/SearchApi.credential.ts @@ -11,6 +11,8 @@ class SearchApi implements INodeCredential { this.label = 'Search API' this.name = 'searchApi' this.version = 1.0 + this.description = + 'Sign in to SearchApi to obtain a free API key from the dashboard.' this.inputs = [ { label: 'SearchApi API Key', From 6cc08110852544fe327271ee121dc821bc115b1d Mon Sep 17 00:00:00 2001 From: Henry Date: Wed, 1 Nov 2023 16:20:13 +0000 Subject: [PATCH 10/12] add check before adding new document --- .../nodes/chains/ConversationChain/ConversationChain.ts | 4 +++- .../components/nodes/vectorstores/Chroma/Chroma_Upsert.ts | 4 +++- .../vectorstores/Elasticsearch/Elasticsearch_Upsert.ts | 4 +++- .../nodes/vectorstores/Faiss_Upsert/Faiss_Upsert.ts | 4 +++- .../nodes/vectorstores/InMemory/InMemoryVectorStore.ts | 2 +- .../components/nodes/vectorstores/Milvus/Milvus_Upsert.ts | 4 +++- .../vectorstores/OpenSearch_Upsert/OpenSearch_Upsert.ts | 4 +++- .../nodes/vectorstores/Pinecone/Pinecone_Upsert.ts | 4 +++- .../nodes/vectorstores/Postgres_Upsert/Postgres_Upsert.ts | 4 +++- .../nodes/vectorstores/Qdrant_Upsert/Qdrant_Upsert.ts | 4 +++- .../components/nodes/vectorstores/Redis/Redis_Upsert.ts | 8 +++++--- .../vectorstores/Singlestore_Upsert/Singlestore_Upsert.ts | 4 +++- .../nodes/vectorstores/Vectara/Vectara_Upsert.ts | 4 +++- .../nodes/vectorstores/Weaviate_Upsert/Weaviate_Upsert.ts | 4 +++- packages/components/nodes/vectorstores/Zep/Zep_Upsert.ts | 4 +++- 15 files changed, 45 insertions(+), 17 deletions(-) diff --git a/packages/components/nodes/chains/ConversationChain/ConversationChain.ts b/packages/components/nodes/chains/ConversationChain/ConversationChain.ts index f12cd30b..92a0b5ea 100644 --- a/packages/components/nodes/chains/ConversationChain/ConversationChain.ts +++ b/packages/components/nodes/chains/ConversationChain/ConversationChain.ts @@ -71,7 +71,9 @@ class ConversationChain_Chains implements INode { const flattenDocs = docs && docs.length ? flatten(docs) : [] const finalDocs = [] for (let i = 0; i < flattenDocs.length; i += 1) { - finalDocs.push(new Document(flattenDocs[i])) + if (flattenDocs[i] && flattenDocs[i].pageContent) { + finalDocs.push(new Document(flattenDocs[i])) + } } let finalText = '' diff --git a/packages/components/nodes/vectorstores/Chroma/Chroma_Upsert.ts b/packages/components/nodes/vectorstores/Chroma/Chroma_Upsert.ts index 0527b729..951338ba 100644 --- a/packages/components/nodes/vectorstores/Chroma/Chroma_Upsert.ts +++ b/packages/components/nodes/vectorstores/Chroma/Chroma_Upsert.ts @@ -98,7 +98,9 @@ class ChromaUpsert_VectorStores implements INode { const flattenDocs = docs && docs.length ? flatten(docs) : [] const finalDocs = [] for (let i = 0; i < flattenDocs.length; i += 1) { - finalDocs.push(new Document(flattenDocs[i])) + if (flattenDocs[i] && flattenDocs[i].pageContent) { + finalDocs.push(new Document(flattenDocs[i])) + } } const obj: { diff --git a/packages/components/nodes/vectorstores/Elasticsearch/Elasticsearch_Upsert.ts b/packages/components/nodes/vectorstores/Elasticsearch/Elasticsearch_Upsert.ts index d4b79a5d..d3965786 100644 --- a/packages/components/nodes/vectorstores/Elasticsearch/Elasticsearch_Upsert.ts +++ b/packages/components/nodes/vectorstores/Elasticsearch/Elasticsearch_Upsert.ts @@ -38,7 +38,9 @@ class ElasicsearchUpsert_VectorStores extends ElasticSearchBase implements INode const flattenDocs = docs && docs.length ? flatten(docs) : [] const finalDocs = [] for (let i = 0; i < flattenDocs.length; i += 1) { - finalDocs.push(new Document(flattenDocs[i])) + if (flattenDocs[i] && flattenDocs[i].pageContent) { + finalDocs.push(new Document(flattenDocs[i])) + } } // The following code is a workaround for a bug (Langchain Issue #1589) in the underlying library. diff --git a/packages/components/nodes/vectorstores/Faiss_Upsert/Faiss_Upsert.ts b/packages/components/nodes/vectorstores/Faiss_Upsert/Faiss_Upsert.ts index c234a4f5..a84b9da4 100644 --- a/packages/components/nodes/vectorstores/Faiss_Upsert/Faiss_Upsert.ts +++ b/packages/components/nodes/vectorstores/Faiss_Upsert/Faiss_Upsert.ts @@ -80,7 +80,9 @@ class FaissUpsert_VectorStores implements INode { const flattenDocs = docs && docs.length ? flatten(docs) : [] const finalDocs = [] for (let i = 0; i < flattenDocs.length; i += 1) { - finalDocs.push(new Document(flattenDocs[i])) + if (flattenDocs[i] && flattenDocs[i].pageContent) { + finalDocs.push(new Document(flattenDocs[i])) + } } const vectorStore = await FaissStore.fromDocuments(finalDocs, embeddings) diff --git a/packages/components/nodes/vectorstores/InMemory/InMemoryVectorStore.ts b/packages/components/nodes/vectorstores/InMemory/InMemoryVectorStore.ts index a480ace0..a827e3ee 100644 --- a/packages/components/nodes/vectorstores/InMemory/InMemoryVectorStore.ts +++ b/packages/components/nodes/vectorstores/InMemory/InMemoryVectorStore.ts @@ -71,7 +71,7 @@ class InMemoryVectorStore_VectorStores implements INode { const flattenDocs = docs && docs.length ? flatten(docs) : [] const finalDocs = [] for (let i = 0; i < flattenDocs.length; i += 1) { - if (flattenDocs[i] !== undefined && flattenDocs[i].pageContent !== undefined) { + if (flattenDocs[i] && flattenDocs[i].pageContent) { finalDocs.push(new Document(flattenDocs[i])) } } diff --git a/packages/components/nodes/vectorstores/Milvus/Milvus_Upsert.ts b/packages/components/nodes/vectorstores/Milvus/Milvus_Upsert.ts index 40afe9a4..371fd16f 100644 --- a/packages/components/nodes/vectorstores/Milvus/Milvus_Upsert.ts +++ b/packages/components/nodes/vectorstores/Milvus/Milvus_Upsert.ts @@ -110,7 +110,9 @@ class Milvus_Upsert_VectorStores implements INode { const flattenDocs = docs && docs.length ? flatten(docs) : [] const finalDocs = [] for (let i = 0; i < flattenDocs.length; i += 1) { - finalDocs.push(new Document(flattenDocs[i])) + if (flattenDocs[i] && flattenDocs[i].pageContent) { + finalDocs.push(new Document(flattenDocs[i])) + } } const vectorStore = await MilvusUpsert.fromDocuments(finalDocs, embeddings, milVusArgs) diff --git a/packages/components/nodes/vectorstores/OpenSearch_Upsert/OpenSearch_Upsert.ts b/packages/components/nodes/vectorstores/OpenSearch_Upsert/OpenSearch_Upsert.ts index c11d8b11..da123581 100644 --- a/packages/components/nodes/vectorstores/OpenSearch_Upsert/OpenSearch_Upsert.ts +++ b/packages/components/nodes/vectorstores/OpenSearch_Upsert/OpenSearch_Upsert.ts @@ -86,7 +86,9 @@ class OpenSearchUpsert_VectorStores implements INode { const flattenDocs = docs && docs.length ? flatten(docs) : [] const finalDocs = [] for (let i = 0; i < flattenDocs.length; i += 1) { - finalDocs.push(new Document(flattenDocs[i])) + if (flattenDocs[i] && flattenDocs[i].pageContent) { + finalDocs.push(new Document(flattenDocs[i])) + } } const client = new Client({ diff --git a/packages/components/nodes/vectorstores/Pinecone/Pinecone_Upsert.ts b/packages/components/nodes/vectorstores/Pinecone/Pinecone_Upsert.ts index 4a12f27b..0851fa3e 100644 --- a/packages/components/nodes/vectorstores/Pinecone/Pinecone_Upsert.ts +++ b/packages/components/nodes/vectorstores/Pinecone/Pinecone_Upsert.ts @@ -106,7 +106,9 @@ class PineconeUpsert_VectorStores implements INode { const flattenDocs = docs && docs.length ? flatten(docs) : [] const finalDocs = [] for (let i = 0; i < flattenDocs.length; i += 1) { - finalDocs.push(new Document(flattenDocs[i])) + if (flattenDocs[i] && flattenDocs[i].pageContent) { + finalDocs.push(new Document(flattenDocs[i])) + } } const obj: PineconeLibArgs = { diff --git a/packages/components/nodes/vectorstores/Postgres_Upsert/Postgres_Upsert.ts b/packages/components/nodes/vectorstores/Postgres_Upsert/Postgres_Upsert.ts index bc16a052..8ea3501d 100644 --- a/packages/components/nodes/vectorstores/Postgres_Upsert/Postgres_Upsert.ts +++ b/packages/components/nodes/vectorstores/Postgres_Upsert/Postgres_Upsert.ts @@ -143,7 +143,9 @@ class PostgresUpsert_VectorStores implements INode { const flattenDocs = docs && docs.length ? flatten(docs) : [] const finalDocs = [] for (let i = 0; i < flattenDocs.length; i += 1) { - finalDocs.push(new Document(flattenDocs[i])) + if (flattenDocs[i] && flattenDocs[i].pageContent) { + finalDocs.push(new Document(flattenDocs[i])) + } } const vectorStore = await TypeORMVectorStore.fromDocuments(finalDocs, embeddings, args) diff --git a/packages/components/nodes/vectorstores/Qdrant_Upsert/Qdrant_Upsert.ts b/packages/components/nodes/vectorstores/Qdrant_Upsert/Qdrant_Upsert.ts index 407a8d22..183271f7 100644 --- a/packages/components/nodes/vectorstores/Qdrant_Upsert/Qdrant_Upsert.ts +++ b/packages/components/nodes/vectorstores/Qdrant_Upsert/Qdrant_Upsert.ts @@ -147,7 +147,9 @@ class QdrantUpsert_VectorStores implements INode { const flattenDocs = docs && docs.length ? flatten(docs) : [] const finalDocs = [] for (let i = 0; i < flattenDocs.length; i += 1) { - finalDocs.push(new Document(flattenDocs[i])) + if (flattenDocs[i] && flattenDocs[i].pageContent) { + finalDocs.push(new Document(flattenDocs[i])) + } } const dbConfig: QdrantLibArgs = { diff --git a/packages/components/nodes/vectorstores/Redis/Redis_Upsert.ts b/packages/components/nodes/vectorstores/Redis/Redis_Upsert.ts index 0ed431fb..9d1a4f45 100644 --- a/packages/components/nodes/vectorstores/Redis/Redis_Upsert.ts +++ b/packages/components/nodes/vectorstores/Redis/Redis_Upsert.ts @@ -49,9 +49,11 @@ class RedisUpsert_VectorStores extends RedisSearchBase implements INode { const flattenDocs = docs && docs.length ? flatten(docs) : [] const finalDocs = [] for (let i = 0; i < flattenDocs.length; i += 1) { - const document = new Document(flattenDocs[i]) - escapeAllStrings(document.metadata) - finalDocs.push(document) + if (flattenDocs[i] && flattenDocs[i].pageContent) { + const document = new Document(flattenDocs[i]) + escapeAllStrings(document.metadata) + finalDocs.push(document) + } } return super.init(nodeData, _, options, flattenDocs) diff --git a/packages/components/nodes/vectorstores/Singlestore_Upsert/Singlestore_Upsert.ts b/packages/components/nodes/vectorstores/Singlestore_Upsert/Singlestore_Upsert.ts index 9889a154..d8edc8d4 100644 --- a/packages/components/nodes/vectorstores/Singlestore_Upsert/Singlestore_Upsert.ts +++ b/packages/components/nodes/vectorstores/Singlestore_Upsert/Singlestore_Upsert.ts @@ -140,7 +140,9 @@ class SingleStoreUpsert_VectorStores implements INode { const flattenDocs = docs && docs.length ? flatten(docs) : [] const finalDocs = [] for (let i = 0; i < flattenDocs.length; i += 1) { - finalDocs.push(new Document(flattenDocs[i])) + if (flattenDocs[i] && flattenDocs[i].pageContent) { + finalDocs.push(new Document(flattenDocs[i])) + } } let vectorStore: SingleStoreVectorStore diff --git a/packages/components/nodes/vectorstores/Vectara/Vectara_Upsert.ts b/packages/components/nodes/vectorstores/Vectara/Vectara_Upsert.ts index 3e035a51..376668de 100644 --- a/packages/components/nodes/vectorstores/Vectara/Vectara_Upsert.ts +++ b/packages/components/nodes/vectorstores/Vectara/Vectara_Upsert.ts @@ -132,7 +132,9 @@ class VectaraUpsert_VectorStores implements INode { const flattenDocs = docs && docs.length ? flatten(docs) : [] const finalDocs = [] for (let i = 0; i < flattenDocs.length; i += 1) { - finalDocs.push(new Document(flattenDocs[i])) + if (flattenDocs[i] && flattenDocs[i].pageContent) { + finalDocs.push(new Document(flattenDocs[i])) + } } const vectorStore = await VectaraStore.fromDocuments(finalDocs, embeddings, vectaraArgs) diff --git a/packages/components/nodes/vectorstores/Weaviate_Upsert/Weaviate_Upsert.ts b/packages/components/nodes/vectorstores/Weaviate_Upsert/Weaviate_Upsert.ts index a2f82831..0b7f2393 100644 --- a/packages/components/nodes/vectorstores/Weaviate_Upsert/Weaviate_Upsert.ts +++ b/packages/components/nodes/vectorstores/Weaviate_Upsert/Weaviate_Upsert.ts @@ -143,7 +143,9 @@ class WeaviateUpsert_VectorStores implements INode { const flattenDocs = docs && docs.length ? flatten(docs) : [] const finalDocs = [] for (let i = 0; i < flattenDocs.length; i += 1) { - finalDocs.push(new Document(flattenDocs[i])) + if (flattenDocs[i] && flattenDocs[i].pageContent) { + finalDocs.push(new Document(flattenDocs[i])) + } } const obj: WeaviateLibArgs = { diff --git a/packages/components/nodes/vectorstores/Zep/Zep_Upsert.ts b/packages/components/nodes/vectorstores/Zep/Zep_Upsert.ts index 0f976d2b..915513b9 100644 --- a/packages/components/nodes/vectorstores/Zep/Zep_Upsert.ts +++ b/packages/components/nodes/vectorstores/Zep/Zep_Upsert.ts @@ -106,7 +106,9 @@ class Zep_Upsert_VectorStores implements INode { const flattenDocs = docs && docs.length ? flatten(docs) : [] const finalDocs = [] for (let i = 0; i < flattenDocs.length; i += 1) { - finalDocs.push(new Document(flattenDocs[i])) + if (flattenDocs[i] && flattenDocs[i].pageContent) { + finalDocs.push(new Document(flattenDocs[i])) + } } const zepConfig: IZepConfig = { From 689612b0d6f79fcc72539453da697dd65877f124 Mon Sep 17 00:00:00 2001 From: Henry Date: Thu, 2 Nov 2023 19:46:52 +0000 Subject: [PATCH 11/12] add query to vec2doc --- .../VectorStoreToDocument.ts | 13 +- .../Prompt Chaining with VectorStore.json | 467 +++++++++--------- packages/ui/src/ui-component/input/Input.js | 76 ++- .../ui/src/views/canvas/NodeInputHandler.js | 3 + 4 files changed, 330 insertions(+), 229 deletions(-) diff --git a/packages/components/nodes/documentloaders/VectorStoreToDocument/VectorStoreToDocument.ts b/packages/components/nodes/documentloaders/VectorStoreToDocument/VectorStoreToDocument.ts index b3f320ce..becd0ac6 100644 --- a/packages/components/nodes/documentloaders/VectorStoreToDocument/VectorStoreToDocument.ts +++ b/packages/components/nodes/documentloaders/VectorStoreToDocument/VectorStoreToDocument.ts @@ -17,7 +17,7 @@ class VectorStoreToDocument_DocumentLoaders implements INode { constructor() { this.label = 'VectorStore To Document' this.name = 'vectorStoreToDocument' - this.version = 1.0 + this.version = 2.0 this.type = 'Document' this.icon = 'vectorretriever.svg' this.category = 'Document Loaders' @@ -29,6 +29,14 @@ class VectorStoreToDocument_DocumentLoaders implements INode { name: 'vectorStore', type: 'VectorStore' }, + { + label: 'Query', + name: 'query', + type: 'string', + description: 'Query to retrieve documents from vector database. If not specified, user question will be used', + optional: true, + acceptVariable: true + }, { label: 'Minimum Score (%)', name: 'minScore', @@ -56,11 +64,12 @@ class VectorStoreToDocument_DocumentLoaders implements INode { async init(nodeData: INodeData, input: string): Promise { const vectorStore = nodeData.inputs?.vectorStore as VectorStore const minScore = nodeData.inputs?.minScore as number + const query = nodeData.inputs?.query as string const output = nodeData.outputs?.output as string const topK = (vectorStore as any)?.k ?? 4 - const docs = await vectorStore.similaritySearchWithScore(input, topK) + const docs = await vectorStore.similaritySearchWithScore(query ?? input, topK) // eslint-disable-next-line no-console console.log('\x1b[94m\x1b[1m\n*****VectorStore Documents*****\n\x1b[0m\x1b[0m') // eslint-disable-next-line no-console diff --git a/packages/server/marketplaces/chatflows/Prompt Chaining with VectorStore.json b/packages/server/marketplaces/chatflows/Prompt Chaining with VectorStore.json index 2af61190..0f270c7b 100644 --- a/packages/server/marketplaces/chatflows/Prompt Chaining with VectorStore.json +++ b/packages/server/marketplaces/chatflows/Prompt Chaining with VectorStore.json @@ -3,120 +3,11 @@ "nodes": [ { "width": 300, - "height": 503, - "id": "pineconeExistingIndex_0", - "position": { - "x": 1062.7418678410986, - "y": -109.27680365777141 - }, - "type": "customNode", - "data": { - "id": "pineconeExistingIndex_0", - "label": "Pinecone Load Existing Index", - "version": 1, - "name": "pineconeExistingIndex", - "type": "Pinecone", - "baseClasses": ["Pinecone", "VectorStoreRetriever", "BaseRetriever"], - "category": "Vector Stores", - "description": "Load existing index from Pinecone (i.e: Document has been upserted)", - "inputParams": [ - { - "label": "Connect Credential", - "name": "credential", - "type": "credential", - "credentialNames": ["pineconeApi"], - "id": "pineconeExistingIndex_0-input-credential-credential" - }, - { - "label": "Pinecone Index", - "name": "pineconeIndex", - "type": "string", - "id": "pineconeExistingIndex_0-input-pineconeIndex-string" - }, - { - "label": "Pinecone Namespace", - "name": "pineconeNamespace", - "type": "string", - "placeholder": "my-first-namespace", - "additionalParams": true, - "optional": true, - "id": "pineconeExistingIndex_0-input-pineconeNamespace-string" - }, - { - "label": "Pinecone Metadata Filter", - "name": "pineconeMetadataFilter", - "type": "json", - "optional": true, - "additionalParams": true, - "id": "pineconeExistingIndex_0-input-pineconeMetadataFilter-json" - }, - { - "label": "Top K", - "name": "topK", - "description": "Number of top results to fetch. Default to 4", - "placeholder": "4", - "type": "number", - "additionalParams": true, - "optional": true, - "id": "pineconeExistingIndex_0-input-topK-number" - } - ], - "inputAnchors": [ - { - "label": "Embeddings", - "name": "embeddings", - "type": "Embeddings", - "id": "pineconeExistingIndex_0-input-embeddings-Embeddings" - } - ], - "inputs": { - "embeddings": "{{openAIEmbeddings_0.data.instance}}", - "pineconeIndex": "newindex", - "pineconeNamespace": "", - "pineconeMetadataFilter": "{}", - "topK": "" - }, - "outputAnchors": [ - { - "name": "output", - "label": "Output", - "type": "options", - "options": [ - { - "id": "pineconeExistingIndex_0-output-retriever-Pinecone|VectorStoreRetriever|BaseRetriever", - "name": "retriever", - "label": "Pinecone Retriever", - "type": "Pinecone | VectorStoreRetriever | BaseRetriever" - }, - { - "id": "pineconeExistingIndex_0-output-vectorStore-Pinecone|VectorStore", - "name": "vectorStore", - "label": "Pinecone Vector Store", - "type": "Pinecone | VectorStore" - } - ], - "default": "retriever" - } - ], - "outputs": { - "output": "vectorStore" - }, - "selected": false - }, - "selected": false, - "positionAbsolute": { - "x": 1062.7418678410986, - "y": -109.27680365777141 - }, - "dragging": false - }, - { - "width": 300, - "height": 327, + "height": 329, "id": "openAIEmbeddings_0", "position": { - "x": 711.3971966563331, - "y": 7.7184225021727 + "x": 1198.6643452533754, + "y": -584.4233173804803 }, "type": "customNode", "data": { @@ -189,18 +80,18 @@ }, "selected": false, "positionAbsolute": { - "x": 711.3971966563331, - "y": 7.7184225021727 + "x": 1198.6643452533754, + "y": -584.4233173804803 }, "dragging": false }, { "width": 300, - "height": 473, + "height": 475, "id": "promptTemplate_0", "position": { - "x": 348.2881107399286, - "y": -97.74510214137423 + "x": 354.2706973608643, + "y": -122.34815000085804 }, "type": "customNode", "data": { @@ -249,18 +140,18 @@ }, "selected": false, "positionAbsolute": { - "x": 348.2881107399286, - "y": -97.74510214137423 + "x": 354.2706973608643, + "y": -122.34815000085804 }, "dragging": false }, { "width": 300, - "height": 522, + "height": 574, "id": "chatOpenAI_0", "position": { - "x": 335.7621848973805, - "y": -721.7411273245009 + "x": 353.5672832154869, + "y": -730.6436764835541 }, "type": "customNode", "data": { @@ -396,7 +287,7 @@ ], "inputs": { "modelName": "gpt-3.5-turbo-16k", - "temperature": 0.9, + "temperature": "0", "maxTokens": "", "topP": "", "frequencyPenalty": "", @@ -418,17 +309,17 @@ "selected": false, "dragging": false, "positionAbsolute": { - "x": 335.7621848973805, - "y": -721.7411273245009 + "x": 353.5672832154869, + "y": -730.6436764835541 } }, { "width": 300, - "height": 522, + "height": 574, "id": "chatOpenAI_1", "position": { - "x": 1765.2801848172305, - "y": -737.9261054149061 + "x": 2281.9246645710673, + "y": -778.8379360672121 }, "type": "customNode", "data": { @@ -564,7 +455,7 @@ ], "inputs": { "modelName": "gpt-3.5-turbo-16k", - "temperature": 0.9, + "temperature": "0", "maxTokens": "", "topP": "", "frequencyPenalty": "", @@ -586,36 +477,153 @@ "selected": false, "dragging": false, "positionAbsolute": { - "x": 1765.2801848172305, - "y": -737.9261054149061 + "x": 2281.9246645710673, + "y": -778.8379360672121 } }, { "width": 300, - "height": 473, - "id": "promptTemplate_1", + "height": 505, + "id": "pineconeExistingIndex_0", "position": { - "x": 1773.720934090435, - "y": -116.71323227575395 + "x": 1544.4998097474581, + "y": -628.8477510577202 }, "type": "customNode", "data": { - "id": "promptTemplate_1", - "label": "Prompt Template", + "id": "pineconeExistingIndex_0", + "label": "Pinecone Load Existing Index", "version": 1, - "name": "promptTemplate", - "type": "PromptTemplate", - "baseClasses": ["PromptTemplate", "BaseStringPromptTemplate", "BasePromptTemplate", "Runnable"], - "category": "Prompts", - "description": "Schema to represent a basic prompt for an LLM", + "name": "pineconeExistingIndex", + "type": "Pinecone", + "baseClasses": ["Pinecone", "VectorStoreRetriever", "BaseRetriever"], + "category": "Vector Stores", + "description": "Load existing index from Pinecone (i.e: Document has been upserted)", "inputParams": [ { - "label": "Template", - "name": "template", + "label": "Connect Credential", + "name": "credential", + "type": "credential", + "credentialNames": ["pineconeApi"], + "id": "pineconeExistingIndex_0-input-credential-credential" + }, + { + "label": "Pinecone Index", + "name": "pineconeIndex", + "type": "string", + "id": "pineconeExistingIndex_0-input-pineconeIndex-string" + }, + { + "label": "Pinecone Namespace", + "name": "pineconeNamespace", + "type": "string", + "placeholder": "my-first-namespace", + "additionalParams": true, + "optional": true, + "id": "pineconeExistingIndex_0-input-pineconeNamespace-string" + }, + { + "label": "Pinecone Metadata Filter", + "name": "pineconeMetadataFilter", + "type": "json", + "optional": true, + "additionalParams": true, + "id": "pineconeExistingIndex_0-input-pineconeMetadataFilter-json" + }, + { + "label": "Top K", + "name": "topK", + "description": "Number of top results to fetch. Default to 4", + "placeholder": "4", + "type": "number", + "additionalParams": true, + "optional": true, + "id": "pineconeExistingIndex_0-input-topK-number" + } + ], + "inputAnchors": [ + { + "label": "Embeddings", + "name": "embeddings", + "type": "Embeddings", + "id": "pineconeExistingIndex_0-input-embeddings-Embeddings" + } + ], + "inputs": { + "embeddings": "{{openAIEmbeddings_0.data.instance}}", + "pineconeIndex": "", + "pineconeNamespace": "", + "pineconeMetadataFilter": "", + "topK": "" + }, + "outputAnchors": [ + { + "name": "output", + "label": "Output", + "type": "options", + "options": [ + { + "id": "pineconeExistingIndex_0-output-retriever-Pinecone|VectorStoreRetriever|BaseRetriever", + "name": "retriever", + "label": "Pinecone Retriever", + "type": "Pinecone | VectorStoreRetriever | BaseRetriever" + }, + { + "id": "pineconeExistingIndex_0-output-vectorStore-Pinecone|VectorStore", + "name": "vectorStore", + "label": "Pinecone Vector Store", + "type": "Pinecone | VectorStore" + } + ], + "default": "retriever" + } + ], + "outputs": { + "output": "vectorStore" + }, + "selected": false + }, + "selected": false, + "positionAbsolute": { + "x": 1544.4998097474581, + "y": -628.8477510577202 + }, + "dragging": false + }, + { + "width": 300, + "height": 652, + "id": "chatPromptTemplate_0", + "position": { + "x": 2290.8365353040026, + "y": -168.49082887954518 + }, + "type": "customNode", + "data": { + "id": "chatPromptTemplate_0", + "label": "Chat Prompt Template", + "version": 1, + "name": "chatPromptTemplate", + "type": "ChatPromptTemplate", + "baseClasses": ["ChatPromptTemplate", "BaseChatPromptTemplate", "BasePromptTemplate", "Runnable"], + "category": "Prompts", + "description": "Schema to represent a chat prompt", + "inputParams": [ + { + "label": "System Message", + "name": "systemMessagePrompt", "type": "string", "rows": 4, - "placeholder": "What is a good name for a company that makes {product}?", - "id": "promptTemplate_1-input-template-string" + "placeholder": "You are a helpful assistant that translates {input_language} to {output_language}.", + "id": "chatPromptTemplate_0-input-systemMessagePrompt-string" + }, + { + "label": "Human Message", + "name": "humanMessagePrompt", + "type": "string", + "rows": 4, + "placeholder": "{text}", + "id": "chatPromptTemplate_0-input-humanMessagePrompt-string" }, { "label": "Format Prompt Values", @@ -624,20 +632,21 @@ "optional": true, "acceptVariable": true, "list": true, - "id": "promptTemplate_1-input-promptValues-json" + "id": "chatPromptTemplate_0-input-promptValues-json" } ], "inputAnchors": [], "inputs": { - "template": "Use the following pieces of context to answer the question at the end.\n\n{context}\n\nQuestion: {question}\nHelpful Answer:", - "promptValues": "{\"context\":\"{{vectorStoreToDocument_0.data.instance}}\",\"question\":\"{{llmChain_0.data.instance}}\"}" + "systemMessagePrompt": "Using the provided context, answer the user's question to the best of your ability using the resources provided. If there is nothing in the context relevant to the question at hand, just say \"Hmm, I'm not sure.\" Don't try to make up an answer.\n\nAnything between the following \\`context\\` html blocks is retrieved from a knowledge bank, not part of the conversation with the user.\n\n\n {context}\n\n\nREMEMBER: If there is no relevant information within the context, just say \"Hmm, I'm not sure.\" Don't try to make up an answer. Anything between the preceding 'context' html blocks is retrieved from a knowledge bank, not part of the conversation with the user.", + "humanMessagePrompt": "{text}", + "promptValues": "{\"context\":\"{{vectorStoreToDocument_0.data.instance}}\",\"text\":\"{{question}}\"}" }, "outputAnchors": [ { - "id": "promptTemplate_1-output-promptTemplate-PromptTemplate|BaseStringPromptTemplate|BasePromptTemplate|Runnable", - "name": "promptTemplate", - "label": "PromptTemplate", - "type": "PromptTemplate | BaseStringPromptTemplate | BasePromptTemplate | Runnable" + "id": "chatPromptTemplate_0-output-chatPromptTemplate-ChatPromptTemplate|BaseChatPromptTemplate|BasePromptTemplate|Runnable", + "name": "chatPromptTemplate", + "label": "ChatPromptTemplate", + "type": "ChatPromptTemplate | BaseChatPromptTemplate | BasePromptTemplate | Runnable" } ], "outputs": {}, @@ -645,18 +654,18 @@ }, "selected": false, "positionAbsolute": { - "x": 1773.720934090435, - "y": -116.71323227575395 + "x": 2290.8365353040026, + "y": -168.49082887954518 }, "dragging": false }, { "width": 300, - "height": 404, + "height": 405, "id": "llmChain_0", "position": { - "x": 756.1670091985342, - "y": -592.5151355056942 + "x": 747.1299875516488, + "y": -267.01184813798244 }, "type": "customNode", "data": { @@ -695,7 +704,7 @@ "inputs": { "model": "{{chatOpenAI_0.data.instance}}", "prompt": "{{promptTemplate_0.data.instance}}", - "chainName": "QuestionChain" + "chainName": "RephraseQuestion" }, "outputAnchors": [ { @@ -726,18 +735,18 @@ }, "selected": false, "positionAbsolute": { - "x": 756.1670091985342, - "y": -592.5151355056942 + "x": 747.1299875516488, + "y": -267.01184813798244 }, "dragging": false }, { "width": 300, - "height": 404, + "height": 405, "id": "llmChain_1", "position": { - "x": 2200.1274896215496, - "y": -144.29167974642334 + "x": 2694.8707655351186, + "y": -308.59150355411236 }, "type": "customNode", "data": { @@ -775,8 +784,8 @@ ], "inputs": { "model": "{{chatOpenAI_1.data.instance}}", - "prompt": "{{promptTemplate_1.data.instance}}", - "chainName": "" + "prompt": "{{chatPromptTemplate_0.data.instance}}", + "chainName": "FinalResponse" }, "outputAnchors": [ { @@ -807,30 +816,39 @@ }, "selected": false, "positionAbsolute": { - "x": 2200.1274896215496, - "y": -144.29167974642334 + "x": 2694.8707655351186, + "y": -308.59150355411236 }, "dragging": false }, { "width": 300, - "height": 353, + "height": 454, "id": "vectorStoreToDocument_0", "position": { - "x": 1407.7038120189868, - "y": -26.16468811205081 + "x": 1906.6871314089658, + "y": -157.0046189166955 }, "type": "customNode", "data": { "id": "vectorStoreToDocument_0", "label": "VectorStore To Document", - "version": 1, + "version": 2, "name": "vectorStoreToDocument", "type": "Document", "baseClasses": ["Document"], "category": "Document Loaders", "description": "Search documents with scores from vector store", "inputParams": [ + { + "label": "Query", + "name": "query", + "type": "string", + "description": "Query to retrieve documents from vector database. If not specified, user question will be used", + "optional": true, + "acceptVariable": true, + "id": "vectorStoreToDocument_0-input-query-string" + }, { "label": "Minimum Score (%)", "name": "minScore", @@ -852,6 +870,7 @@ ], "inputs": { "vectorStore": "{{pineconeExistingIndex_0.data.instance}}", + "query": "{{llmChain_0.data.instance}}", "minScore": "" }, "outputAnchors": [ @@ -883,8 +902,8 @@ }, "selected": false, "positionAbsolute": { - "x": 1407.7038120189868, - "y": -26.16468811205081 + "x": 1906.6871314089658, + "y": -157.0046189166955 }, "dragging": false } @@ -901,6 +920,50 @@ "label": "" } }, + { + "source": "promptTemplate_0", + "sourceHandle": "promptTemplate_0-output-promptTemplate-PromptTemplate|BaseStringPromptTemplate|BasePromptTemplate|Runnable", + "target": "llmChain_0", + "targetHandle": "llmChain_0-input-prompt-BasePromptTemplate", + "type": "buttonedge", + "id": "promptTemplate_0-promptTemplate_0-output-promptTemplate-PromptTemplate|BaseStringPromptTemplate|BasePromptTemplate|Runnable-llmChain_0-llmChain_0-input-prompt-BasePromptTemplate", + "data": { + "label": "" + } + }, + { + "source": "chatOpenAI_0", + "sourceHandle": "chatOpenAI_0-output-chatOpenAI-ChatOpenAI|BaseChatModel|BaseLanguageModel|Runnable", + "target": "llmChain_0", + "targetHandle": "llmChain_0-input-model-BaseLanguageModel", + "type": "buttonedge", + "id": "chatOpenAI_0-chatOpenAI_0-output-chatOpenAI-ChatOpenAI|BaseChatModel|BaseLanguageModel|Runnable-llmChain_0-llmChain_0-input-model-BaseLanguageModel", + "data": { + "label": "" + } + }, + { + "source": "chatPromptTemplate_0", + "sourceHandle": "chatPromptTemplate_0-output-chatPromptTemplate-ChatPromptTemplate|BaseChatPromptTemplate|BasePromptTemplate|Runnable", + "target": "llmChain_1", + "targetHandle": "llmChain_1-input-prompt-BasePromptTemplate", + "type": "buttonedge", + "id": "chatPromptTemplate_0-chatPromptTemplate_0-output-chatPromptTemplate-ChatPromptTemplate|BaseChatPromptTemplate|BasePromptTemplate|Runnable-llmChain_1-llmChain_1-input-prompt-BasePromptTemplate", + "data": { + "label": "" + } + }, + { + "source": "chatOpenAI_1", + "sourceHandle": "chatOpenAI_1-output-chatOpenAI-ChatOpenAI|BaseChatModel|BaseLanguageModel|Runnable", + "target": "llmChain_1", + "targetHandle": "llmChain_1-input-model-BaseLanguageModel", + "type": "buttonedge", + "id": "chatOpenAI_1-chatOpenAI_1-output-chatOpenAI-ChatOpenAI|BaseChatModel|BaseLanguageModel|Runnable-llmChain_1-llmChain_1-input-model-BaseLanguageModel", + "data": { + "label": "" + } + }, { "source": "pineconeExistingIndex_0", "sourceHandle": "pineconeExistingIndex_0-output-vectorStore-Pinecone|VectorStore", @@ -915,32 +978,10 @@ { "source": "vectorStoreToDocument_0", "sourceHandle": "vectorStoreToDocument_0-output-text-string|json", - "target": "promptTemplate_1", - "targetHandle": "promptTemplate_1-input-promptValues-json", + "target": "chatPromptTemplate_0", + "targetHandle": "chatPromptTemplate_0-input-promptValues-json", "type": "buttonedge", - "id": "vectorStoreToDocument_0-vectorStoreToDocument_0-output-text-string|json-promptTemplate_1-promptTemplate_1-input-promptValues-json", - "data": { - "label": "" - } - }, - { - "source": "chatOpenAI_0", - "sourceHandle": "chatOpenAI_0-output-chatOpenAI-ChatOpenAI|BaseChatModel|BaseLanguageModel|Runnable", - "target": "llmChain_0", - "targetHandle": "llmChain_0-input-model-BaseLanguageModel", - "type": "buttonedge", - "id": "chatOpenAI_0-chatOpenAI_0-output-chatOpenAI-ChatOpenAI|BaseChatModel|BaseLanguageModel|Runnable-llmChain_0-llmChain_0-input-model-BaseLanguageModel", - "data": { - "label": "" - } - }, - { - "source": "promptTemplate_0", - "sourceHandle": "promptTemplate_0-output-promptTemplate-PromptTemplate|BaseStringPromptTemplate|BasePromptTemplate|Runnable", - "target": "llmChain_0", - "targetHandle": "llmChain_0-input-prompt-BasePromptTemplate", - "type": "buttonedge", - "id": "promptTemplate_0-promptTemplate_0-output-promptTemplate-PromptTemplate|BaseStringPromptTemplate|BasePromptTemplate|Runnable-llmChain_0-llmChain_0-input-prompt-BasePromptTemplate", + "id": "vectorStoreToDocument_0-vectorStoreToDocument_0-output-text-string|json-chatPromptTemplate_0-chatPromptTemplate_0-input-promptValues-json", "data": { "label": "" } @@ -948,32 +989,10 @@ { "source": "llmChain_0", "sourceHandle": "llmChain_0-output-outputPrediction-string|json", - "target": "promptTemplate_1", - "targetHandle": "promptTemplate_1-input-promptValues-json", + "target": "vectorStoreToDocument_0", + "targetHandle": "vectorStoreToDocument_0-input-query-string", "type": "buttonedge", - "id": "llmChain_0-llmChain_0-output-outputPrediction-string|json-promptTemplate_1-promptTemplate_1-input-promptValues-json", - "data": { - "label": "" - } - }, - { - "source": "chatOpenAI_1", - "sourceHandle": "chatOpenAI_1-output-chatOpenAI-ChatOpenAI|BaseChatModel|BaseLanguageModel|Runnable", - "target": "llmChain_1", - "targetHandle": "llmChain_1-input-model-BaseLanguageModel", - "type": "buttonedge", - "id": "chatOpenAI_1-chatOpenAI_1-output-chatOpenAI-ChatOpenAI|BaseChatModel|BaseLanguageModel|Runnable-llmChain_1-llmChain_1-input-model-BaseLanguageModel", - "data": { - "label": "" - } - }, - { - "source": "promptTemplate_1", - "sourceHandle": "promptTemplate_1-output-promptTemplate-PromptTemplate|BaseStringPromptTemplate|BasePromptTemplate|Runnable", - "target": "llmChain_1", - "targetHandle": "llmChain_1-input-prompt-BasePromptTemplate", - "type": "buttonedge", - "id": "promptTemplate_1-promptTemplate_1-output-promptTemplate-PromptTemplate|BaseStringPromptTemplate|BasePromptTemplate|Runnable-llmChain_1-llmChain_1-input-prompt-BasePromptTemplate", + "id": "llmChain_0-llmChain_0-output-outputPrediction-string|json-vectorStoreToDocument_0-vectorStoreToDocument_0-input-query-string", "data": { "label": "" } diff --git a/packages/ui/src/ui-component/input/Input.js b/packages/ui/src/ui-component/input/Input.js index 95bf968d..6993847b 100644 --- a/packages/ui/src/ui-component/input/Input.js +++ b/packages/ui/src/ui-component/input/Input.js @@ -1,10 +1,39 @@ -import { useState } from 'react' +import { useState, useEffect, useRef } from 'react' import PropTypes from 'prop-types' -import { FormControl, OutlinedInput } from '@mui/material' +import { FormControl, OutlinedInput, Popover } from '@mui/material' import ExpandTextDialog from 'ui-component/dialog/ExpandTextDialog' +import SelectVariable from 'ui-component/json/SelectVariable' +import { getAvailableNodesForVariable } from 'utils/genericHelper' -export const Input = ({ inputParam, value, onChange, disabled = false, showDialog, dialogProps, onDialogCancel, onDialogConfirm }) => { +export const Input = ({ + inputParam, + value, + nodes, + edges, + nodeId, + onChange, + disabled = false, + showDialog, + dialogProps, + onDialogCancel, + onDialogConfirm +}) => { const [myValue, setMyValue] = useState(value ?? '') + const [anchorEl, setAnchorEl] = useState(null) + const [availableNodesForVariable, setAvailableNodesForVariable] = useState([]) + const ref = useRef(null) + + const openPopOver = Boolean(anchorEl) + + const handleClosePopOver = () => { + setAnchorEl(null) + } + + const setNewVal = (val) => { + const newVal = myValue + val.substring(2) + onChange(newVal) + setMyValue(newVal) + } const getInputType = (type) => { switch (type) { @@ -19,6 +48,19 @@ export const Input = ({ inputParam, value, onChange, disabled = false, showDialo } } + useEffect(() => { + if (!disabled && nodes && edges && nodeId && inputParam) { + const nodesForVariable = inputParam?.acceptVariable ? getAvailableNodesForVariable(nodes, edges, nodeId, inputParam.id) : [] + setAvailableNodesForVariable(nodesForVariable) + } + }, [disabled, inputParam, nodes, edges, nodeId]) + + useEffect(() => { + if (typeof myValue === 'string' && myValue && myValue.endsWith('{{')) { + setAnchorEl(ref.current) + } + }, [myValue]) + return ( <> @@ -55,6 +97,31 @@ export const Input = ({ inputParam, value, onChange, disabled = false, showDialo }} > )} +
+ {inputParam?.acceptVariable && ( + + { + setNewVal(val) + handleClosePopOver() + }} + /> + + )} ) } @@ -66,6 +133,9 @@ Input.propTypes = { disabled: PropTypes.bool, showDialog: PropTypes.bool, dialogProps: PropTypes.object, + nodes: PropTypes.array, + edges: PropTypes.array, + nodeId: PropTypes.string, onDialogCancel: PropTypes.func, onDialogConfirm: PropTypes.func } diff --git a/packages/ui/src/views/canvas/NodeInputHandler.js b/packages/ui/src/views/canvas/NodeInputHandler.js index 3a13a3b5..028c0d36 100644 --- a/packages/ui/src/views/canvas/NodeInputHandler.js +++ b/packages/ui/src/views/canvas/NodeInputHandler.js @@ -265,6 +265,9 @@ const NodeInputHandler = ({ inputAnchor, inputParam, data, disabled = false, isA inputParam={inputParam} onChange={(newValue) => (data.inputs[inputParam.name] = newValue)} value={data.inputs[inputParam.name] ?? inputParam.default ?? ''} + nodes={inputParam?.acceptVariable && reactFlowInstance ? reactFlowInstance.getNodes() : []} + edges={inputParam?.acceptVariable && reactFlowInstance ? reactFlowInstance.getEdges() : []} + nodeId={data.id} showDialog={showExpandDialog} dialogProps={expandDialogProps} onDialogCancel={() => setShowExpandDialog(false)} From a240333e794b950c1e7c98d2a48be631c885fc67 Mon Sep 17 00:00:00 2001 From: SebastjanPrachovskij Date: Thu, 2 Nov 2023 22:17:36 +0200 Subject: [PATCH 12/12] Run yarn lint-fix --- .../credentials/SearchApi.credential.ts | 2 +- .../documentloaders/SearchApi/SearchAPI.ts | 6 +-- .../chatflows/Chat with a Podcast.json | 53 ++++--------------- 3 files changed, 14 insertions(+), 47 deletions(-) diff --git a/packages/components/credentials/SearchApi.credential.ts b/packages/components/credentials/SearchApi.credential.ts index 96d3dc01..f23577a1 100644 --- a/packages/components/credentials/SearchApi.credential.ts +++ b/packages/components/credentials/SearchApi.credential.ts @@ -12,7 +12,7 @@ class SearchApi implements INodeCredential { this.name = 'searchApi' this.version = 1.0 this.description = - 'Sign in to SearchApi to obtain a free API key from the dashboard.' + 'Sign in to SearchApi to obtain a free API key from the dashboard.' this.inputs = [ { label: 'SearchApi API Key', diff --git a/packages/components/nodes/documentloaders/SearchApi/SearchAPI.ts b/packages/components/nodes/documentloaders/SearchApi/SearchAPI.ts index 9d14dc5a..b7f0b0b5 100644 --- a/packages/components/nodes/documentloaders/SearchApi/SearchAPI.ts +++ b/packages/components/nodes/documentloaders/SearchApi/SearchAPI.ts @@ -74,17 +74,17 @@ class SearchAPI_DocumentLoaders implements INode { const searchApiKey = getCredentialParam('searchApiKey', credentialData, nodeData) // Check and parse custom parameters (should be JSON or object) - const parsedParameters = typeof customParameters === 'object' ? customParameters : JSON.parse(customParameters || '{}'); + const parsedParameters = typeof customParameters === 'object' ? customParameters : JSON.parse(customParameters || '{}') // Prepare the configuration for the SearchApiLoader const loaderConfig = { q: query, apiKey: searchApiKey, ...parsedParameters - }; + } // Initialize the loader with the given configuration - const loader = new SearchApiLoader(loaderConfig); + const loader = new SearchApiLoader(loaderConfig) // Fetch documents, split if a text splitter is provided const docs = textSplitter ? await loader.loadAndSplit() : await loader.load() diff --git a/packages/server/marketplaces/chatflows/Chat with a Podcast.json b/packages/server/marketplaces/chatflows/Chat with a Podcast.json index e91bf50f..2a9e05b5 100644 --- a/packages/server/marketplaces/chatflows/Chat with a Podcast.json +++ b/packages/server/marketplaces/chatflows/Chat with a Podcast.json @@ -16,11 +16,7 @@ "version": 1, "name": "conversationalRetrievalQAChain", "type": "ConversationalRetrievalQAChain", - "baseClasses": [ - "ConversationalRetrievalQAChain", - "BaseChain", - "Runnable" - ], + "baseClasses": ["ConversationalRetrievalQAChain", "BaseChain", "Runnable"], "category": "Chains", "description": "Document QA - built on RetrievalQAChain to provide a chat history component", "inputParams": [ @@ -130,11 +126,7 @@ "version": 1, "name": "memoryVectorStore", "type": "Memory", - "baseClasses": [ - "Memory", - "VectorStoreRetriever", - "BaseRetriever" - ], + "baseClasses": ["Memory", "VectorStoreRetriever", "BaseRetriever"], "category": "Vector Stores", "description": "In-memory vectorstore that stores embeddings and does an exact, linear search for the most similar embeddings.", "inputParams": [ @@ -164,11 +156,7 @@ } ], "inputs": { - "document": [ - "{{searchApi_0.data.instance}}", - "{{searchApi_0.data.instance}}", - "{{searchApi_0.data.instance}}" - ], + "document": ["{{searchApi_0.data.instance}}", "{{searchApi_0.data.instance}}", "{{searchApi_0.data.instance}}"], "embeddings": "{{openAIEmbeddings_0.data.instance}}", "topK": "" }, @@ -221,12 +209,7 @@ "version": 2, "name": "chatOpenAI", "type": "ChatOpenAI", - "baseClasses": [ - "ChatOpenAI", - "BaseChatModel", - "BaseLanguageModel", - "Runnable" - ], + "baseClasses": ["ChatOpenAI", "BaseChatModel", "BaseLanguageModel", "Runnable"], "category": "Chat Models", "description": "Wrapper around OpenAI large language models that use the Chat endpoint", "inputParams": [ @@ -234,9 +217,7 @@ "label": "Connect Credential", "name": "credential", "type": "credential", - "credentialNames": [ - "openAIApi" - ], + "credentialNames": ["openAIApi"], "id": "chatOpenAI_0-input-credential-credential" }, { @@ -406,12 +387,7 @@ "version": 1, "name": "characterTextSplitter", "type": "CharacterTextSplitter", - "baseClasses": [ - "CharacterTextSplitter", - "TextSplitter", - "BaseDocumentTransformer", - "Runnable" - ], + "baseClasses": ["CharacterTextSplitter", "TextSplitter", "BaseDocumentTransformer", "Runnable"], "category": "Text Splitters", "description": "splits only on one type of character (defaults to \"\\n\\n\").", "inputParams": [ @@ -479,10 +455,7 @@ "version": 1, "name": "openAIEmbeddings", "type": "OpenAIEmbeddings", - "baseClasses": [ - "OpenAIEmbeddings", - "Embeddings" - ], + "baseClasses": ["OpenAIEmbeddings", "Embeddings"], "category": "Embeddings", "description": "OpenAI API to generate embeddings for a given text", "inputParams": [ @@ -490,9 +463,7 @@ "label": "Connect Credential", "name": "credential", "type": "credential", - "credentialNames": [ - "openAIApi" - ], + "credentialNames": ["openAIApi"], "id": "openAIEmbeddings_0-input-credential-credential" }, { @@ -568,9 +539,7 @@ "version": 1, "name": "searchApi", "type": "Document", - "baseClasses": [ - "Document" - ], + "baseClasses": ["Document"], "category": "Document Loaders", "description": "Load data from real-time search results", "inputParams": [ @@ -579,9 +548,7 @@ "name": "credential", "type": "credential", "optional": false, - "credentialNames": [ - "searchApi" - ], + "credentialNames": ["searchApi"], "id": "searchApi_0-input-credential-credential" }, {