From d588ac0480846271e00324caf413cd408eb5d19e Mon Sep 17 00:00:00 2001 From: vinodkiran Date: Fri, 29 Sep 2023 07:54:16 +0530 Subject: [PATCH 01/25] Adding support for LLM Caching. --- .../LocalMemoryCache/LocalMemoryCache.ts | 44 ++++++++++++++++++ .../llmcache/LocalMemoryCache/memorycache.png | Bin 0 -> 2717 bytes .../components/nodes/llms/OpenAI/OpenAI.ts | 17 +++++-- packages/components/package.json | 2 +- packages/components/src/Interface.ts | 7 +++ packages/components/src/handler.ts | 22 ++++++++- 6 files changed, 87 insertions(+), 5 deletions(-) create mode 100644 packages/components/nodes/llmcache/LocalMemoryCache/LocalMemoryCache.ts create mode 100644 packages/components/nodes/llmcache/LocalMemoryCache/memorycache.png diff --git a/packages/components/nodes/llmcache/LocalMemoryCache/LocalMemoryCache.ts b/packages/components/nodes/llmcache/LocalMemoryCache/LocalMemoryCache.ts new file mode 100644 index 00000000..73f4415e --- /dev/null +++ b/packages/components/nodes/llmcache/LocalMemoryCache/LocalMemoryCache.ts @@ -0,0 +1,44 @@ +import { INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface' +import { InMemoryCache } from 'langchain/cache' +import { getBaseClasses } from '../../../src' + +class LocalMemoryCache implements INode { + label: string + name: string + version: number + description: string + type: string + icon: string + category: string + baseClasses: string[] + inputs: INodeParams[] + outputs: INodeOutputsValue[] + inMemoryCache: any + + constructor() { + this.label = 'Local (Builtin) Cache' + this.name = 'localCache' + this.version = 1.0 + this.type = 'LLMCache' + this.icon = 'memorycache.png' + this.category = 'LLM Cache' + this.baseClasses = [this.type, 'LLMCacheBase'] + this.inputs = [] + this.outputs = [ + { + label: 'LLM Cache', + name: 'cache', + baseClasses: [this.type, ...getBaseClasses(InMemoryCache)] + } + ] + } + + async init(nodeData: INodeData): Promise { + if (!this.inMemoryCache) { + this.inMemoryCache = InMemoryCache.global() + } + return this.inMemoryCache + } +} + +module.exports = { nodeClass: LocalMemoryCache } diff --git a/packages/components/nodes/llmcache/LocalMemoryCache/memorycache.png b/packages/components/nodes/llmcache/LocalMemoryCache/memorycache.png new file mode 100644 index 0000000000000000000000000000000000000000..aaeecd6fb100be3a10b3d1faca046f9fa409f018 GIT binary patch literal 2717 zcmV;O3S#w%P)Px#1ZP1_K>z@;j|==^1poj532;bRa{vG=N&o;XN&$CzbWH#N3NlGVK~!i%-C283 zROc1{vCA&(gIyj10)qh}C=crxZQ5yO635s{6UQ-WYc%!I^ba*Q6JNDcCz}4D@m1TI zDEO$24-_rdM;v3EJghMqry4;Ul?7k;Ho7411s0a2=bU@*x*~*YY%KOSbGUoI``zz5 z_q@+XNIL;4s%vTxYBZ{MJrB%kXlN*`R;%0#^WWUOiTMi_;ABw|!otI8mOSfmVCD^> zM$CKX9Sk2nTw#9c(xsR)XATl$tbE&FA1gRoC+yb-NOwtd7(PR=8ndoY!dz0A|AS0PpIBCcn zri@!?F;-76UK}kgK0Bo z!qeEOjy^b8>rSTwKOH&@&CSj7K}MrNVV<~_32pv1?B2BtrLGd3JA01q{n)fQAJI|K zICku)GL=wUU5&gIEBG~(Uo3}-?2r*`9=slJQgplsx$iAUTU(pVe^Br%Y%Xo>ZNhl4 z`HIcYeS02Ou6Pgeaq(y*X|;G;#8fP#s)mLJ`3N=v4Qx)H`g*j}LTny<-zj6XPfJUS zXy9RCpuKE%yD)$2)-8N}=Cnwwea7?|y!B&6B_*zQk$h=cDO|2na#oY_s3(IY-{|pR z#E21!*;G-qsjaOg+f`klhl9gqT52i@nlSCHw?q}>cI{-Q4@4AeJ01VNi1}o4m6Y(_ zk1rl;<^!Q}Ro)6TH#JMF=9o+>>K1;|U{qw3WIcZGQGoZkefu^}m7Gx|&#|+H;`a8P zJJ9TFMQghsEjq|Y3ypf^Wq2qTJW6PSQY1fj9{#jwAzt{|ix@a)5Sl2Jv*eXYA@Xrl zUdam#9_KO6dHz%pnR0XJJGZ(A#a#2mv*8`_b$#jf}hzKd>9uljYbzQ_TlKU?U0t2z z429}YM^~oWI=u~bK2~<{AqPC_MUawNR@>;x_(O?vZci0_>h;pJ& zaxz}|#m_Nn)XSu)-V&`|{pDz7al-Lq$KY@})w_GqHS@8=d{9&o(B#_sIxPFkpVa~2 z)iGl+W%6X~-mwb}-X=Jxld_ZJvUUvABv2e8=9Rg8k=By0q(!!ME|aUPs?kcBB0cR% zEL^e}qetImKBr$M>Kp3CpE@F#`ED~TZ)>7XD=T|IfUaD=gn}(wgz;EvRH9>Iq%D*V zhoo#UDB)Ei5)s=o6Mr1rxShuFS&AuvuvM{jA?niPw&C<0fEhVWBj~Jm=#13)Dioq(N@? zw~NDagU-_C^Pr6W{gIxYE)L1Ke(f5LQhS-dF(2#Juahx`Dbyt9)*W;)&kM0BXyB-{ zV96qE+K`W!=okzbI1nkRDa2F^o}eo`VWcw>5e^4@lmj?KdKx`A{q-q)`pGA_b@Lku z!&k3f!S(Amkd%}t#~BZbZXK4@*XU`)$NN81M}XawsB$T^hFQXqkd%b75*MuD7SaYs z7(UNwho4l>j>^5KX7>!f=d&JmY%McKMMvW3SI6ZyV?1f}%P$Vzqiqe$k8X$U46K8C z3U8C9yn&+PVr-;1%^5T@(kT@tOP!sV%_EAgoP>CcpK%C`a70KB363T#GL7^?Kl7>t znQCflSOhqKwhV4}1&$m(3@b4c0MAnq_;q*aJ~Y&O&_d>=kzVL#o-0j_CjWM-oL+6) zwhd{?eXw!;1~}NhDVecanL2{UIL`B&77BwNN_rfDuHC$WpFHynob`E1e$9mwtcRJ1`ip6 zDO0E7`i+~|w534ZMT0)$(G_hNY@-?$zq<&RE|p{AZ(hgW*1s=fEX6CAFT-8orjE=@ zq#QJH0N}Gtn_8$PB_<)Q?~`O%ArgfOHgCqELx-_$!+M#|FdFr-rJH$<4jMT8ZKVvm zch4R)QmW6%%tU5(4zjbesJcu>X8%mYQ=ukiPJevfR-B>uUsiS&A5mg+opGV0)I}w8 z4g6#xadELQ2NiEYF7vI>@32APfTM(+B$<$yh}S1h5L4N+`vbf?e?ERaegd8vGMFTs zh(xA@MD$5{0x!MvBBs6hCf2U`D~gMb6VpCKI-KGw;TDT%B2ZH3gf8Ygv%sdtCXAmj z5v$j(Mc$g#l1&q+mQP7ZL3(N$GN|ARroJhuNTzBshC(MtCa!b3X7s-Duv6%ld=JF` zKl5Cnvzj$(@YKt(_rZe)q)J>v%}FF>PLAWj@jRHiivL`ZvdQ`Q9hHpgd4G4%#XMCh zxNzY@fZia-OYheYpa1g!)#bhM*~g#4VzUZ!n$u}DyhhqM{{w^1>p8S?jlv%`a^y&< z@in@7`D&+)NJn%VDtSYW(|?>dR~-Qg3jYpQS*e^PaQjzWTm)B1sZ@zRbxv1^u-K5% zF9SIPaxm}*1Mvbyp{%Sd%$xhRIs;U3mLZCo`R>qt8>oVtW(^J0OguMqD3&f;MvY0P zjIlJi7vhHY=;|b{$E#FSs4yf zG3WKQ!eTNbp1La5t{yx*^D0C-v>0oJCC?>+r={!O(?lP^O$R3>{$Cc`wdUk{aUb(@ z{xAot@^ZU8`p^URjo@on(ANL1`|x|$sC_IU?FsYRf4~!Z#XXSZA2nDotYJPM{pv)q z!aR5I3l}ZIN$QHs<}fjZo(Mlh0IM||vq{j;4;!X1uRfE*QN51yzW|>|goT?G0R#9C XQt;1#;KI))00000NkvXXu0mjfl~Wzs literal 0 HcmV?d00001 diff --git a/packages/components/nodes/llms/OpenAI/OpenAI.ts b/packages/components/nodes/llms/OpenAI/OpenAI.ts index 2960ad2a..9fa61653 100644 --- a/packages/components/nodes/llms/OpenAI/OpenAI.ts +++ b/packages/components/nodes/llms/OpenAI/OpenAI.ts @@ -1,6 +1,8 @@ import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface' import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils' import { OpenAI, OpenAIInput } from 'langchain/llms/openai' +import { BaseLLMParams } from 'langchain/dist/llms/base' +import { BaseCache } from 'langchain/schema' class OpenAI_LLMs implements INode { label: string @@ -17,7 +19,7 @@ class OpenAI_LLMs implements INode { constructor() { this.label = 'OpenAI' this.name = 'openAI' - this.version = 2.0 + this.version = 3.0 this.type = 'OpenAI' this.icon = 'openai.png' this.category = 'LLMs' @@ -30,6 +32,12 @@ class OpenAI_LLMs implements INode { credentialNames: ['openAIApi'] } this.inputs = [ + { + label: 'Cache', + name: 'llmCache', + type: 'LLMCache', + optional: true + }, { label: 'Model Name', name: 'modelName', @@ -149,7 +157,9 @@ class OpenAI_LLMs implements INode { const credentialData = await getCredentialData(nodeData.credential ?? '', options) const openAIApiKey = getCredentialParam('openAIApiKey', credentialData, nodeData) - const obj: Partial & { openAIApiKey?: string } = { + const llmCache = nodeData.inputs?.llmCache as BaseCache + + const obj: Partial & BaseLLMParams & { openAIApiKey?: string } = { temperature: parseFloat(temperature), modelName, openAIApiKey, @@ -164,8 +174,9 @@ class OpenAI_LLMs implements INode { if (batchSize) obj.batchSize = parseInt(batchSize, 10) if (bestOf) obj.bestOf = parseInt(bestOf, 10) - let parsedBaseOptions: any | undefined = undefined + if (llmCache) obj.cache = llmCache + let parsedBaseOptions: any | undefined = undefined if (baseOptions) { try { parsedBaseOptions = typeof baseOptions === 'object' ? baseOptions : JSON.parse(baseOptions) diff --git a/packages/components/package.json b/packages/components/package.json index 93609106..f8498e31 100644 --- a/packages/components/package.json +++ b/packages/components/package.json @@ -42,7 +42,7 @@ "google-auth-library": "^9.0.0", "graphql": "^16.6.0", "html-to-text": "^9.0.5", - "langchain": "^0.0.152", + "langchain": "^0.0.154", "langfuse-langchain": "^1.0.14-alpha.0", "langsmith": "^0.0.32", "linkifyjs": "^4.1.1", diff --git a/packages/components/src/Interface.ts b/packages/components/src/Interface.ts index e883d056..76dc7354 100644 --- a/packages/components/src/Interface.ts +++ b/packages/components/src/Interface.ts @@ -1,6 +1,7 @@ /** * Types */ +import { BaseCache } from 'langchain/schema' export type NodeParamsType = | 'asyncOptions' @@ -176,3 +177,9 @@ export class VectorStoreRetriever { this.vectorStore = fields.vectorStore } } + +export interface LLMCacheBase { + name: string + description: string + baseCache: BaseCache +} diff --git a/packages/components/src/handler.ts b/packages/components/src/handler.ts index 10f9a214..a102b473 100644 --- a/packages/components/src/handler.ts +++ b/packages/components/src/handler.ts @@ -151,6 +151,7 @@ export class CustomChainHandler extends BaseCallbackHandler { socketIOClientId = '' skipK = 0 // Skip streaming for first K numbers of handleLLMStart returnSourceDocuments = false + cachedResponse = true constructor(socketIO: Server, socketIOClientId: string, skipK?: number, returnSourceDocuments?: boolean) { super() @@ -161,6 +162,7 @@ export class CustomChainHandler extends BaseCallbackHandler { } handleLLMStart() { + this.cachedResponse = false if (this.skipK > 0) this.skipK -= 1 } @@ -175,13 +177,31 @@ export class CustomChainHandler extends BaseCallbackHandler { } handleLLMEnd() { - this.socketIO.to(this.socketIOClientId).emit('end') + /* send the end event from handleChainEnd */ + // this.socketIO.to(this.socketIOClientId).emit('end') } handleChainEnd(outputs: ChainValues): void | Promise { if (this.returnSourceDocuments) { this.socketIO.to(this.socketIOClientId).emit('sourceDocuments', outputs?.sourceDocuments) } + /* + Langchain does not call handleLLMStart, handleLLMEnd, handleLLMNewToken when the chain is cached. + Callback Order is "Chain Start -> LLM Start --> LLM Token --> LLM End -> Chain End" for normal responses. + Callback Order is "Chain Start -> Chain End" for cached responses. + */ + if (this.cachedResponse) { + const cachedValue = outputs.text as string + //split at whitespace, and keep the whitespace. This is to preserve the original formatting. + const result = cachedValue.split(/(\s+)/) + result.forEach((token: string, index: number) => { + if (index === 0) { + this.socketIO.to(this.socketIOClientId).emit('start', token) + } + this.socketIO.to(this.socketIOClientId).emit('token', token) + }) + } + this.socketIO.to(this.socketIOClientId).emit('end') } } From d81869fd59de2894f15b1649a909d4e513393302 Mon Sep 17 00:00:00 2001 From: vinodkiran Date: Fri, 29 Sep 2023 08:19:44 +0530 Subject: [PATCH 02/25] Extending support for Caching to all LLM's. --- .../nodes/llms/Azure OpenAI/AzureOpenAI.ts | 16 +++++++++++++--- .../components/nodes/llms/Bittensor/Bittensor.ts | 15 +++++++++++++-- packages/components/nodes/llms/Cohere/Cohere.ts | 13 ++++++++++--- .../nodes/llms/GooglePaLM/GooglePaLM.ts | 12 ++++++++++-- .../nodes/llms/GoogleVertexAI/GoogleVertexAI.ts | 11 ++++++++++- .../HuggingFaceInference/HuggingFaceInference.ts | 13 ++++++++++++- packages/components/nodes/llms/OpenAI/OpenAI.ts | 2 +- .../components/nodes/llms/Replicate/Replicate.ts | 16 ++++++++++++++-- 8 files changed, 83 insertions(+), 15 deletions(-) diff --git a/packages/components/nodes/llms/Azure OpenAI/AzureOpenAI.ts b/packages/components/nodes/llms/Azure OpenAI/AzureOpenAI.ts index f48c4642..e5cca769 100644 --- a/packages/components/nodes/llms/Azure OpenAI/AzureOpenAI.ts +++ b/packages/components/nodes/llms/Azure OpenAI/AzureOpenAI.ts @@ -1,7 +1,8 @@ import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface' import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils' import { AzureOpenAIInput, OpenAI, OpenAIInput } from 'langchain/llms/openai' - +import { BaseCache } from 'langchain/schema' +import { BaseLLMParams } from 'langchain/llms/base' class AzureOpenAI_LLMs implements INode { label: string name: string @@ -17,7 +18,7 @@ class AzureOpenAI_LLMs implements INode { constructor() { this.label = 'Azure OpenAI' this.name = 'azureOpenAI' - this.version = 1.0 + this.version = 2.0 this.type = 'AzureOpenAI' this.icon = 'Azure.svg' this.category = 'LLMs' @@ -30,6 +31,12 @@ class AzureOpenAI_LLMs implements INode { credentialNames: ['azureOpenAIApi'] } this.inputs = [ + { + label: 'Cache', + name: 'llmCache', + type: 'LLMCache', + optional: true + }, { label: 'Model Name', name: 'modelName', @@ -163,7 +170,9 @@ class AzureOpenAI_LLMs implements INode { const azureOpenAIApiDeploymentName = getCredentialParam('azureOpenAIApiDeploymentName', credentialData, nodeData) const azureOpenAIApiVersion = getCredentialParam('azureOpenAIApiVersion', credentialData, nodeData) - const obj: Partial & Partial = { + const llmCache = nodeData.inputs?.llmCache as BaseCache + + const obj: Partial & BaseLLMParams & Partial = { temperature: parseFloat(temperature), modelName, azureOpenAIApiKey, @@ -179,6 +188,7 @@ class AzureOpenAI_LLMs implements INode { if (presencePenalty) obj.presencePenalty = parseFloat(presencePenalty) if (timeout) obj.timeout = parseInt(timeout, 10) if (bestOf) obj.bestOf = parseInt(bestOf, 10) + if (llmCache) obj.cache = llmCache const model = new OpenAI(obj) return model diff --git a/packages/components/nodes/llms/Bittensor/Bittensor.ts b/packages/components/nodes/llms/Bittensor/Bittensor.ts index a87a7e48..84401e59 100644 --- a/packages/components/nodes/llms/Bittensor/Bittensor.ts +++ b/packages/components/nodes/llms/Bittensor/Bittensor.ts @@ -1,6 +1,8 @@ import { INode, INodeData, INodeParams } from '../../../src/Interface' import { getBaseClasses } from '../../../src/utils' import { NIBittensorLLM, BittensorInput } from 'langchain/experimental/llms/bittensor' +import { BaseCache } from 'langchain/schema' +import { BaseLLMParams } from 'langchain/llms/base' class Bittensor_LLMs implements INode { label: string @@ -16,13 +18,19 @@ class Bittensor_LLMs implements INode { constructor() { this.label = 'NIBittensorLLM' this.name = 'NIBittensorLLM' - this.version = 1.0 + this.version = 2.0 this.type = 'Bittensor' this.icon = 'logo.png' this.category = 'LLMs' this.description = 'Wrapper around Bittensor subnet 1 large language models' this.baseClasses = [this.type, ...getBaseClasses(NIBittensorLLM)] this.inputs = [ + { + label: 'Cache', + name: 'llmCache', + type: 'LLMCache', + optional: true + }, { label: 'System prompt', name: 'system_prompt', @@ -44,10 +52,13 @@ class Bittensor_LLMs implements INode { async init(nodeData: INodeData, _: string): Promise { const system_prompt = nodeData.inputs?.system_prompt as string const topResponses = Number(nodeData.inputs?.topResponses as number) - const obj: Partial = { + const llmCache = nodeData.inputs?.llmCache as BaseCache + + const obj: Partial & BaseLLMParams = { systemPrompt: system_prompt, topResponses: topResponses } + if (llmCache) obj.cache = llmCache const model = new NIBittensorLLM(obj) return model diff --git a/packages/components/nodes/llms/Cohere/Cohere.ts b/packages/components/nodes/llms/Cohere/Cohere.ts index 4a3a8a80..8b4c0ac2 100644 --- a/packages/components/nodes/llms/Cohere/Cohere.ts +++ b/packages/components/nodes/llms/Cohere/Cohere.ts @@ -1,6 +1,7 @@ import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface' import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils' import { Cohere, CohereInput } from './core' +import { BaseCache } from 'langchain/schema' class Cohere_LLMs implements INode { label: string @@ -17,7 +18,7 @@ class Cohere_LLMs implements INode { constructor() { this.label = 'Cohere' this.name = 'cohere' - this.version = 1.0 + this.version = 2.0 this.type = 'Cohere' this.icon = 'cohere.png' this.category = 'LLMs' @@ -30,6 +31,12 @@ class Cohere_LLMs implements INode { credentialNames: ['cohereApi'] } this.inputs = [ + { + label: 'Cache', + name: 'llmCache', + type: 'LLMCache', + optional: true + }, { label: 'Model Name', name: 'modelName', @@ -85,7 +92,7 @@ class Cohere_LLMs implements INode { const temperature = nodeData.inputs?.temperature as string const modelName = nodeData.inputs?.modelName as string const maxTokens = nodeData.inputs?.maxTokens as string - + const llmCache = nodeData.inputs?.llmCache as BaseCache const credentialData = await getCredentialData(nodeData.credential ?? '', options) const cohereApiKey = getCredentialParam('cohereApiKey', credentialData, nodeData) @@ -96,7 +103,7 @@ class Cohere_LLMs implements INode { if (maxTokens) obj.maxTokens = parseInt(maxTokens, 10) if (modelName) obj.model = modelName if (temperature) obj.temperature = parseFloat(temperature) - + if (llmCache) obj.cache = llmCache const model = new Cohere(obj) return model } diff --git a/packages/components/nodes/llms/GooglePaLM/GooglePaLM.ts b/packages/components/nodes/llms/GooglePaLM/GooglePaLM.ts index 24630360..d916c09f 100644 --- a/packages/components/nodes/llms/GooglePaLM/GooglePaLM.ts +++ b/packages/components/nodes/llms/GooglePaLM/GooglePaLM.ts @@ -1,7 +1,7 @@ import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface' import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils' import { GooglePaLM, GooglePaLMTextInput } from 'langchain/llms/googlepalm' - +import { BaseCache } from 'langchain/schema' class GooglePaLM_LLMs implements INode { label: string name: string @@ -17,7 +17,7 @@ class GooglePaLM_LLMs implements INode { constructor() { this.label = 'GooglePaLM' this.name = 'GooglePaLM' - this.version = 1.0 + this.version = 2.0 this.type = 'GooglePaLM' this.icon = 'Google_PaLM_Logo.svg' this.category = 'LLMs' @@ -30,6 +30,12 @@ class GooglePaLM_LLMs implements INode { credentialNames: ['googleMakerSuite'] } this.inputs = [ + { + label: 'Cache', + name: 'llmCache', + type: 'LLMCache', + optional: true + }, { label: 'Model Name', name: 'modelName', @@ -126,6 +132,7 @@ class GooglePaLM_LLMs implements INode { const topP = nodeData.inputs?.topP as string const topK = nodeData.inputs?.topK as string const stopSequencesObj = nodeData.inputs?.stopSequencesObj + const llmCache = nodeData.inputs?.llmCache as BaseCache const credentialData = await getCredentialData(nodeData.credential ?? '', options) const googleMakerSuiteKey = getCredentialParam('googleMakerSuiteKey', credentialData, nodeData) @@ -139,6 +146,7 @@ class GooglePaLM_LLMs implements INode { if (maxOutputTokens) obj.maxOutputTokens = parseInt(maxOutputTokens, 10) if (topP) obj.topP = parseFloat(topP) if (topK) obj.topK = parseFloat(topK) + if (llmCache) obj.cache = llmCache let parsedStopSequences: any | undefined = undefined if (stopSequencesObj) { diff --git a/packages/components/nodes/llms/GoogleVertexAI/GoogleVertexAI.ts b/packages/components/nodes/llms/GoogleVertexAI/GoogleVertexAI.ts index 4d19d04f..41b18475 100644 --- a/packages/components/nodes/llms/GoogleVertexAI/GoogleVertexAI.ts +++ b/packages/components/nodes/llms/GoogleVertexAI/GoogleVertexAI.ts @@ -2,6 +2,7 @@ import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Inter import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils' import { GoogleVertexAI, GoogleVertexAITextInput } from 'langchain/llms/googlevertexai' import { GoogleAuthOptions } from 'google-auth-library' +import { BaseCache } from 'langchain/schema' class GoogleVertexAI_LLMs implements INode { label: string @@ -18,7 +19,7 @@ class GoogleVertexAI_LLMs implements INode { constructor() { this.label = 'GoogleVertexAI' this.name = 'googlevertexai' - this.version = 1.0 + this.version = 2.0 this.type = 'GoogleVertexAI' this.icon = 'vertexai.svg' this.category = 'LLMs' @@ -34,6 +35,12 @@ class GoogleVertexAI_LLMs implements INode { 'Google Vertex AI credential. If you are using a GCP service like Cloud Run, or if you have installed default credentials on your local machine, you do not need to set this credential.' } this.inputs = [ + { + label: 'Cache', + name: 'llmCache', + type: 'LLMCache', + optional: true + }, { label: 'Model Name', name: 'modelName', @@ -120,6 +127,7 @@ class GoogleVertexAI_LLMs implements INode { const modelName = nodeData.inputs?.modelName as string const maxOutputTokens = nodeData.inputs?.maxOutputTokens as string const topP = nodeData.inputs?.topP as string + const llmCache = nodeData.inputs?.llmCache as BaseCache const obj: Partial = { temperature: parseFloat(temperature), @@ -129,6 +137,7 @@ class GoogleVertexAI_LLMs implements INode { if (maxOutputTokens) obj.maxOutputTokens = parseInt(maxOutputTokens, 10) if (topP) obj.topP = parseFloat(topP) + if (llmCache) obj.cache = llmCache const model = new GoogleVertexAI(obj) return model diff --git a/packages/components/nodes/llms/HuggingFaceInference/HuggingFaceInference.ts b/packages/components/nodes/llms/HuggingFaceInference/HuggingFaceInference.ts index c7f6a37e..9d6226e2 100644 --- a/packages/components/nodes/llms/HuggingFaceInference/HuggingFaceInference.ts +++ b/packages/components/nodes/llms/HuggingFaceInference/HuggingFaceInference.ts @@ -1,6 +1,7 @@ import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface' import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils' import { HFInput, HuggingFaceInference } from './core' +import { BaseCache } from 'langchain/schema' class HuggingFaceInference_LLMs implements INode { label: string @@ -17,7 +18,7 @@ class HuggingFaceInference_LLMs implements INode { constructor() { this.label = 'HuggingFace Inference' this.name = 'huggingFaceInference_LLMs' - this.version = 1.0 + this.version = 2.0 this.type = 'HuggingFaceInference' this.icon = 'huggingface.png' this.category = 'LLMs' @@ -30,6 +31,12 @@ class HuggingFaceInference_LLMs implements INode { credentialNames: ['huggingFaceApi'] } this.inputs = [ + { + label: 'Cache', + name: 'llmCache', + type: 'LLMCache', + optional: true + }, { label: 'Model', name: 'model', @@ -106,6 +113,8 @@ class HuggingFaceInference_LLMs implements INode { const credentialData = await getCredentialData(nodeData.credential ?? '', options) const huggingFaceApiKey = getCredentialParam('huggingFaceApiKey', credentialData, nodeData) + const llmCache = nodeData.inputs?.llmCache as BaseCache + const obj: Partial = { model, apiKey: huggingFaceApiKey @@ -119,6 +128,8 @@ class HuggingFaceInference_LLMs implements INode { if (endpoint) obj.endpoint = endpoint const huggingFace = new HuggingFaceInference(obj) + if (llmCache) huggingFace.cache = llmCache + return huggingFace } } diff --git a/packages/components/nodes/llms/OpenAI/OpenAI.ts b/packages/components/nodes/llms/OpenAI/OpenAI.ts index 9fa61653..607ebeb4 100644 --- a/packages/components/nodes/llms/OpenAI/OpenAI.ts +++ b/packages/components/nodes/llms/OpenAI/OpenAI.ts @@ -1,7 +1,7 @@ import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface' import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils' import { OpenAI, OpenAIInput } from 'langchain/llms/openai' -import { BaseLLMParams } from 'langchain/dist/llms/base' +import { BaseLLMParams } from 'langchain/llms/base' import { BaseCache } from 'langchain/schema' class OpenAI_LLMs implements INode { diff --git a/packages/components/nodes/llms/Replicate/Replicate.ts b/packages/components/nodes/llms/Replicate/Replicate.ts index 22c6e93a..fd1eb6f7 100644 --- a/packages/components/nodes/llms/Replicate/Replicate.ts +++ b/packages/components/nodes/llms/Replicate/Replicate.ts @@ -1,6 +1,8 @@ import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface' import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils' import { Replicate, ReplicateInput } from 'langchain/llms/replicate' +import { BaseCache } from 'langchain/schema' +import { BaseLLMParams } from 'langchain/llms/base' class Replicate_LLMs implements INode { label: string @@ -17,7 +19,7 @@ class Replicate_LLMs implements INode { constructor() { this.label = 'Replicate' this.name = 'replicate' - this.version = 1.0 + this.version = 2.0 this.type = 'Replicate' this.icon = 'replicate.svg' this.category = 'LLMs' @@ -30,6 +32,12 @@ class Replicate_LLMs implements INode { credentialNames: ['replicateApi'] } this.inputs = [ + { + label: 'Cache', + name: 'llmCache', + type: 'LLMCache', + optional: true + }, { label: 'Model', name: 'model', @@ -103,7 +111,9 @@ class Replicate_LLMs implements INode { const name = modelName.split(':')[0].split('/').pop() const org = modelName.split(':')[0].split('/')[0] - const obj: ReplicateInput = { + const llmCache = nodeData.inputs?.llmCache as BaseCache + + const obj: ReplicateInput & BaseLLMParams = { model: `${org}/${name}:${version}`, apiKey } @@ -120,6 +130,8 @@ class Replicate_LLMs implements INode { } if (Object.keys(inputs).length) obj.input = inputs + if (llmCache) obj.cache = llmCache + const model = new Replicate(obj) return model } From 18702e4c479916ac409fd136d1b13aba85a6ed29 Mon Sep 17 00:00:00 2001 From: vinodkiran Date: Sat, 30 Sep 2023 12:05:51 +0530 Subject: [PATCH 03/25] Refactoring of LLM In Cache Memory files for better clarity of name --- .../LLMInMemoryCache.ts} | 4 ++-- .../memorycache.png | Bin 2 files changed, 2 insertions(+), 2 deletions(-) rename packages/components/nodes/llmcache/{LocalMemoryCache/LocalMemoryCache.ts => LLMInMemoryCache/LLMInMemoryCache.ts} (92%) rename packages/components/nodes/llmcache/{LocalMemoryCache => LLMInMemoryCache}/memorycache.png (100%) diff --git a/packages/components/nodes/llmcache/LocalMemoryCache/LocalMemoryCache.ts b/packages/components/nodes/llmcache/LLMInMemoryCache/LLMInMemoryCache.ts similarity index 92% rename from packages/components/nodes/llmcache/LocalMemoryCache/LocalMemoryCache.ts rename to packages/components/nodes/llmcache/LLMInMemoryCache/LLMInMemoryCache.ts index 73f4415e..5e77a029 100644 --- a/packages/components/nodes/llmcache/LocalMemoryCache/LocalMemoryCache.ts +++ b/packages/components/nodes/llmcache/LLMInMemoryCache/LLMInMemoryCache.ts @@ -2,7 +2,7 @@ import { INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/I import { InMemoryCache } from 'langchain/cache' import { getBaseClasses } from '../../../src' -class LocalMemoryCache implements INode { +class LLMInMemoryCache implements INode { label: string name: string version: number @@ -41,4 +41,4 @@ class LocalMemoryCache implements INode { } } -module.exports = { nodeClass: LocalMemoryCache } +module.exports = { nodeClass: LLMInMemoryCache } diff --git a/packages/components/nodes/llmcache/LocalMemoryCache/memorycache.png b/packages/components/nodes/llmcache/LLMInMemoryCache/memorycache.png similarity index 100% rename from packages/components/nodes/llmcache/LocalMemoryCache/memorycache.png rename to packages/components/nodes/llmcache/LLMInMemoryCache/memorycache.png From 4efec94dc8d404755379a8068628341870c5da04 Mon Sep 17 00:00:00 2001 From: vinodkiran Date: Sat, 30 Sep 2023 12:07:48 +0530 Subject: [PATCH 04/25] LLM Cache - Addition of Momento Cache --- .../credentials/MomentoCacheApi.credential.ts | 36 +++++++++ .../LLMMomentoCache/LLMMomentoCache.ts | 75 ++++++++++++++++++ .../llmcache/LLMMomentoCache/momento.png | Bin 0 -> 5534 bytes packages/components/package.json | 1 + 4 files changed, 112 insertions(+) create mode 100644 packages/components/credentials/MomentoCacheApi.credential.ts create mode 100644 packages/components/nodes/llmcache/LLMMomentoCache/LLMMomentoCache.ts create mode 100644 packages/components/nodes/llmcache/LLMMomentoCache/momento.png diff --git a/packages/components/credentials/MomentoCacheApi.credential.ts b/packages/components/credentials/MomentoCacheApi.credential.ts new file mode 100644 index 00000000..038f826d --- /dev/null +++ b/packages/components/credentials/MomentoCacheApi.credential.ts @@ -0,0 +1,36 @@ +import { INodeParams, INodeCredential } from '../src/Interface' + +class MomentoCacheApi implements INodeCredential { + label: string + name: string + version: number + description: string + inputs: INodeParams[] + + constructor() { + this.label = 'Momento Cache API' + this.name = 'momentoCacheApi' + this.version = 1.0 + this.description = + 'Refer to official guide on how to get API key on Momento' + this.inputs = [ + { + label: 'Cache', + name: 'momentoCache', + type: 'string' + }, + { + label: 'API Key', + name: 'momentoApiKey', + type: 'password' + }, + { + label: 'Endpoint', + name: 'momentoEndpoint', + type: 'string' + } + ] + } +} + +module.exports = { credClass: MomentoCacheApi } diff --git a/packages/components/nodes/llmcache/LLMMomentoCache/LLMMomentoCache.ts b/packages/components/nodes/llmcache/LLMMomentoCache/LLMMomentoCache.ts new file mode 100644 index 00000000..82267c24 --- /dev/null +++ b/packages/components/nodes/llmcache/LLMMomentoCache/LLMMomentoCache.ts @@ -0,0 +1,75 @@ +import { + getBaseClasses, + getCredentialData, + getCredentialParam, + ICommonObject, + INode, + INodeData, + INodeOutputsValue, + INodeParams +} from '../../../src' +import { MomentoCache } from 'langchain/cache/momento' +import { CacheClient, Configurations, CredentialProvider } from '@gomomento/sdk' + +class LLMMomentoCache implements INode { + label: string + name: string + version: number + description: string + type: string + icon: string + category: string + baseClasses: string[] + inputs: INodeParams[] + outputs: INodeOutputsValue[] + credential: INodeParams + + constructor() { + this.label = 'Momento Cache' + this.name = 'momentoCache' + this.version = 1.0 + this.type = 'LLMCache' + this.icon = 'momento.png' + this.category = 'LLM Cache' + this.baseClasses = [this.type, 'LLMCacheBase'] + this.credential = { + label: 'Connect Credential', + name: 'credential', + type: 'credential', + optional: true, + credentialNames: ['momentoCacheApi'] + } + this.inputs = [] + this.outputs = [ + { + label: 'LLM Cache', + name: 'cache', + baseClasses: [this.type, ...getBaseClasses(MomentoCache)] + } + ] + } + + async init(nodeData: INodeData, _: string, options: ICommonObject): Promise { + const credentialData = await getCredentialData(nodeData.credential ?? '', options) + const apiKey = getCredentialParam('momentoApiKey', credentialData, nodeData) + const cacheName = getCredentialParam('momentoCache', credentialData, nodeData) + const endPoint = getCredentialParam('momentoEndpoint', credentialData, nodeData) + + // See https://github.com/momentohq/client-sdk-javascript for connection options + const client = new CacheClient({ + configuration: Configurations.Laptop.v1(), + credentialProvider: CredentialProvider.fromString({ + apiKey: apiKey + }), + defaultTtlSeconds: 60 * 60 * 24 + }) + + let momentoCache = await MomentoCache.fromProps({ + client, + cacheName: cacheName + }) + return momentoCache + } +} + +module.exports = { nodeClass: LLMMomentoCache } diff --git a/packages/components/nodes/llmcache/LLMMomentoCache/momento.png b/packages/components/nodes/llmcache/LLMMomentoCache/momento.png new file mode 100644 index 0000000000000000000000000000000000000000..0f2b54b6a0c0e6100d2546d053e0b4548c42e4c6 GIT binary patch literal 5534 zcma)=XH*kWw}$BksUcLUp@kMu5RobggetuX5~TMI5Ae?05CI_+>AizU zmm*+91w`ZGqdK*nLRUS=DhpaFHT=qo#y(D>qJCEG@2T!hQNOP-wL7t zwn}fNf`A>dpP@REsCJBF4>%xmLg*lfh#C^9{@9QM$CTb05B!LT#I*jc#DPKnIz&WF zQktp=V+{H*$Hm=bcD}zw;>;`w5!M7^r2=b7myVDP=M&>bY@y&Vdad{ZjR^)Ew~!!n z%u5ir7IwB+jWkUa=O8)6$U{%!96LF{AKm~$A&5!&U;Q}rGmm=vdUJ?MuJB`g@^g)#k5h)%oFhqjcqnKPnRaR z&e!H>X=@vC%K7WpSTq}YEWb7R^gb{>kEav?b|ndK3;J0p=eH9t7kKnq&SxWXlUwQO z!mSSjBO}HU6f}l=dwVZF*I3HT7OB(CR+&ktH}PYH@I=aWzozx>RSXSj{rnX8x#IHT zw2C|A4AJRE%f`m$y4DrudiHI7zu_a7pX$>4K(BLg0#)q9wXs~q1JKc0cf|g(*N30Y zPE_Gvzq-FlP2K#F-xE#aN4C@!;C3#q$rL+5Ix$@kxZLEDYSH4EQ(0S^b{v^b*!)=K zv)M1(;rJosa?2ovn$afZVXHUdaH(0_W3jOhjGh@B9W|LQ)iL5zi(li_57&>R3}qUs z`|#JI2Hml=caD|MuoOqPo9pDry$X*fjlD_x>+52X(%&zqJ zoNa?C`q-NkED~tr?d@&Xb$rx!kS5az-cy4>I+S@Ey8LwVit4R#-mH8pV-ZyXa8 zkbm-9Tg^u^r9M)MU=`@06*Zqaf&&kM*Y!~;BzKl_8mq3Uu6|iXufr>WAI+ATcXV`Y z3sH9NexR%S!Vg}kVINb(F5_uDe($Qi2`l`_foVDn2Z0Uot;2lFgzD?&-#F=ApU|hj zC+{-FKilGE?PMayaQD)NtgF4~2)`;#U6|=%*B@V@81$q3VV^fc5cu!UdF$pUPoC7W zWH=^nfB7en+QH3E#$CT6gDLr{!S2LF0 zfXeeq0WK{nB9ao=dW-t0#)|IiyU^p@D>8AU|EcHIZFoT3&+dwf3ZKg-7AMTcxzoxh z?2x+WmDnh|8|?_>Q%DT+vVO53hy-F|V)E+Xp4LgfMQn@$=3D;3@nlvD4@oB}7#gkO z;^INW3sUYpJUdq zJQKvb&DXyB%Xj-2o-9jcsL-l-4X)Hfl0JVXh%7Ayjp;|+LVtVXIgW_?+}_UuHZ?YG z?ZjlArsd?okAwc)w7)6clUtRJ5l61NL^Rv)2(xy!DYC#rRMv22KqCe9HMha1taq(X z``htG;P15T?FjwBt{|?8Y_rMRxwxdj`8D{f1+^o#AUpL$u2+G#pD4s-RJ<3(s)uj% zu~;)o_~>U7m-K167W>k8^4Z>}Ts8U08V)riud4DK*NOYQfm`;V)oZLNc)0lY@}dM4 zj*rkT12m_+&3}&=q^V@*yIGL(_j8E}PAc-YP+**eo_W zYzU}L`62la0fDb+(wN|XvS>jTg=uVTEJiVuc&e_iOfW$Aa=m}<&u%2=4~~zsj0b-s zu4Ff@0A}tmA3r!{sZvERLuddQWnHp&i)`S6#xUjPguJY_Kj}STFt78%Vd8XngFOpT zN*K(RL8ey^hQ#yQW3tf00|QB-V2)j;2I#206~U6X!v^n%pPDixU}+skvjouI`H#8r zZ}jCEsOSU)1axI@F=THKr^n4pQr5gEjGgK@T@{5FlFm#@p@__(_xyL0>>L~%#C%rC z#JmHJSFJ2Y*M7wKghMGqqDsfZwMKKPF&=k6o12>tWX#CLot7IR6>a8gtsab7@GU5H zc~3CYrw6Td5eIx*L%{0!V&6G!#tH`%vtGMjRQowLE^ga(fzmX<=Nr55jiMA9h1_rW zR@A^(m*1m_{=sr#LJi~$JGRn5+WY>wU$AyvDDiy;Lgb@^`x=lK_Aib1u_TW zNuoDZqN$V9s9rEyhDwwFv@B%rvLOogU>#fJdN*y@SKX>Q;HvanM;#uJs@z4SiaYJ@ zj}HO);!ttC_KK8JV#%O*p6=JRHnK+sMQSClkDy%pMHu(R>n=x%E(Vbc2j0Pk7WIxd zW@l%0OL$x6ud84xtn?B9h~Y$OvLp&AKfIM9kCvi&bWUS z!Y5-x1a)5v<+AbnI@Z=+qaU}fcm1=SZY}3iQ`#7CU`oj1vGtEqF_01)DAI-}Ye<-T zq31lX*NmjP>9`ck(KdEjOa;r3$#CiF``N0J$woNFTR*RZJ|8yk-f4lljI0asZHztw zlIKPyv92!vkk^MxLA3W;F%HMp57u6OnZdi#U>Ac?g>CjHK*p(4a0{AxdMpy=t05O2 zo4(sm34JUZ(#k=-@4G_iI(4SqaK`I2;r8FD$Q!&^sUTlsbbOeg5ZO$dg&3)1)!|BM z7B~Z0smHEN4Rk|}!-2tEDnsd68^)2a8hnhKmjw0Bar)$AD zd~$4xK}{IP}u{k0Wqeh}koXUUf@U$%CRw^iFH|F!BdIHJqy z)|Tkt&*k*ro#$a&xSl1Ai*zrU-1fB-!~5;1)hMrFNH*UZ4a4l~qAF4>sjT*csC zsThDf?pm|s>d~31+yD&#QIwU8AQg#nuKY+(z1=U1PI*kg)c|*hZztJaxX;AX<@O$q*G?D5srwT~q{e}ETVR%MWDfpIrRtH-uP$3Ksw{l!FBpM) z$FoWGNo*S8oI9&HI(ZFHXGC5)6#dNa14jL8F1=khj1R1*ugd~5D^quf9b zl+=vb8OfaDMT8X!+lWVS@;i;^nxqQqq@FS0XWH#MlteI*r=DUA~Y1TOsisM zX7(A53~k41n5l=`0A#z7Y}|z<$v(LQ@Mwa(jhpAAD3^<0hr0Q~A*}xUGlLcI&|X;} zkP!grb*XYhPT&Rt*1nGaR@OT*eqBHm9UM%t0ME*E6Is{48+e}NXu*VDR@OQ%eWK?f zS+*1RN1`*lP%uET_#h73w*Cw{Xkcu-^*eXTen5g93t|r!kx*R) z5fvTY^$Qy;Y=*gmqU{G1w#DYvK&5xwGyCFLs({3=A^KF|$ApR_mOkH7&GlP&ILQ#>{# zL)QE6O-)dNp{Bi@)uC#L(6!EsBcrfmh#>OQ#7uyf0CjX>)|Mx5`NPk3^*0KKvJyC@ z!^-kPmIfGiDw1+?MpcK!d5j+CDLpG@qS^P9bEH*gZ*-X!B;ZNM-P<62UrFec&hpQB zdnw?BZg?_Uz103kPztAmLbZ=CIJ55!*)#krCD`w{C#8LwnVG2vIg{S})YZfnyVh@) z00N6*6`7V1%E%tW!L2#;19Mvhq))mds^XEIJoPg3eTPZbEsBInB0{EGQmnKhRJlv0 zf8tI=mcJNDP+U0+6e$iXPH8BGk=bAi%ZQG@t;cZ3vgkz9a_&EE){6YEwk}?45xnun zdmTwA)O2(#J(-mQZ}djhbWo1zr&zkLv>LSU+SY)&59&UM=UUyIz2wX|w5ss@)Fx|P zW06tuvHzQ(SWn))=V*&biU*CsKX#!tyM_{qqQOXNO^;!{p7S-%&foi@ghK6}a$!*v z$X%PJTckbk?CYEH-Z*iZGQqjniMKalrH!rRzk-q{75Axc+}2QtL!AxqC1RbFu<`$x zrKwQI!Jz~*DagqC^v9%npHyD?ag6lVM1jf%D*Ncu`#7ft5I1Ixm@pD!2?<yeF_ zEZO7DY35Z8=de`nclM3rlz)t6u~JLqg0I4>nbt2)KTG5;@@TN!s@pQHQ1m_h*SU$D zLrt*d3^>;pcSom3To{>$+sFCHR9033&+7S8T?m=CH(!^B*#cCIQKGGtJX1eXDJ45w zl#3Y4mLS&?L&tJV%UFdm~6SQ#VHr#{y3Z=y+G;&R9Tq^Hr3*jgVau^(YMr2u*t5Vf}^9 zwT6!3665s$IZb!(Pd#Q5n)_RrvEU;QGPgo(WpH4CUP5jpN8YS`K|hkK=lqd-k{e9x z5kfGRT+R)J*_#p|jNsoI$0H)>9E~vtH7&hvTtNHDjt*gDV-S|dnAm#DbLB1pPg5%J zk6KJm%teuqP5O`VY@3YF_#1Fq`WFD(NT7$kb$6lF_+?20*{#H;O>+KK`09G&Gcw#%BK8lmaN~gONGGu*vbTWk~VS8`= zV_neEyDwaXAK~&mS|Q23yNhqi{y7v)Unwra|3SH_t$BxB>`P+7o58=_2^zd&KuND~ zasFdkuTPr;@D5lj(>RREbeYCI=_NS4{Nve86T@ZSzVY4GBBb=V zf`9GLYAbZ89{Pu-xBuR++E~=|@$ng&|GXcSNyQ@Wtd|*D@(V-j$q-ndKigoZ?y^X( z6bx7Z>89V!KS4HVi!CETOM-(DO9^jtyZ`KDd-ZD5RG~oVkaGvZk1mn+ilU1ff+`3^ zm{a$zDV=(E8r*WGSTnVlNzB~%AFDKJh-5T=VV97WmaYsqn48j~IS9rl^Qp(OC{UZ6 z&}QQo0KHX&@YJItGmLh%GJ%k*HrlN0%rwS=u>X0e()~jvdv2vtoQnELD12yqcG>+9p+^e*+fM}Q=J6lgoK5CI>ak1(CFMhNFbe=t8IDJA7*R%79?B~7!)k~;qWJ80lU%_5%l)a6I0%$jWr#Xb4=mr6HUlt)2~fxzEt7bJy8Z5j=-th} y60{!I0Q|&M(6$m3dI$0Uv={%k11YL?c~5Gk5mTY&umE_fM4BjF)mr4E@c#hjL3Jbm literal 0 HcmV?d00001 diff --git a/packages/components/package.json b/packages/components/package.json index f8498e31..18bf8e6a 100644 --- a/packages/components/package.json +++ b/packages/components/package.json @@ -19,6 +19,7 @@ "@aws-sdk/client-dynamodb": "^3.360.0", "@dqbd/tiktoken": "^1.0.7", "@getzep/zep-js": "^0.6.3", + "@gomomento/sdk": "^1.40.2", "@google-ai/generativelanguage": "^0.2.1", "@huggingface/inference": "^2.6.1", "@notionhq/client": "^2.2.8", From 29be2d2608de2e7c21b762041409aab8b827cb7d Mon Sep 17 00:00:00 2001 From: vinodkiran Date: Sat, 30 Sep 2023 18:51:14 +0530 Subject: [PATCH 05/25] Addition of Redis LLM Cache --- .../credentials/RedisCacheApi.credential.ts | 43 +++++++++++ .../llmcache/LLMRedisCache/LLMRedisCache.ts | 75 +++++++++++++++++++ .../nodes/llmcache/LLMRedisCache/redis.svg | 1 + packages/components/package.json | 1 + 4 files changed, 120 insertions(+) create mode 100644 packages/components/credentials/RedisCacheApi.credential.ts create mode 100644 packages/components/nodes/llmcache/LLMRedisCache/LLMRedisCache.ts create mode 100644 packages/components/nodes/llmcache/LLMRedisCache/redis.svg diff --git a/packages/components/credentials/RedisCacheApi.credential.ts b/packages/components/credentials/RedisCacheApi.credential.ts new file mode 100644 index 00000000..e09a94e7 --- /dev/null +++ b/packages/components/credentials/RedisCacheApi.credential.ts @@ -0,0 +1,43 @@ +import { INodeParams, INodeCredential } from '../src/Interface' + +class RedisCacheApi implements INodeCredential { + label: string + name: string + version: number + description: string + inputs: INodeParams[] + + constructor() { + this.label = 'Redis Cache API' + this.name = 'redisCacheApi' + this.version = 1.0 + this.inputs = [ + { + label: 'Redis Host', + name: 'redisCacheHost', + type: 'string', + default: '127.0.0.1' + }, + { + label: 'Port', + name: 'redisCachePort', + type: 'number', + default: '6789' + }, + { + label: 'User', + name: 'redisCacheUser', + type: 'string', + placeholder: '' + }, + { + label: 'Password', + name: 'redisCachePwd', + type: 'password', + placeholder: '' + } + ] + } +} + +module.exports = { credClass: RedisCacheApi } diff --git a/packages/components/nodes/llmcache/LLMRedisCache/LLMRedisCache.ts b/packages/components/nodes/llmcache/LLMRedisCache/LLMRedisCache.ts new file mode 100644 index 00000000..4a098e82 --- /dev/null +++ b/packages/components/nodes/llmcache/LLMRedisCache/LLMRedisCache.ts @@ -0,0 +1,75 @@ +import { + getBaseClasses, + getCredentialData, + getCredentialParam, + ICommonObject, + INode, + INodeData, + INodeOutputsValue, + INodeParams +} from '../../../src' +import { RedisCache } from 'langchain/cache/ioredis' +import { Redis } from 'ioredis' + +class LLMRedisCache implements INode { + label: string + name: string + version: number + description: string + type: string + icon: string + category: string + baseClasses: string[] + inputs: INodeParams[] + outputs: INodeOutputsValue[] + credential: INodeParams + + constructor() { + this.label = 'Redis LLM Cache' + this.name = 'redisCache' + this.version = 1.0 + this.type = 'LLMCache' + this.icon = 'redis.svg' + this.category = 'LLM Cache' + this.baseClasses = [this.type, 'LLMCacheBase'] + this.credential = { + label: 'Connect Credential', + name: 'credential', + type: 'credential', + optional: true, + credentialNames: ['redisCacheApi'] + } + this.inputs = [] + this.outputs = [ + { + label: 'LLM Cache', + name: 'cache', + baseClasses: [this.type, ...getBaseClasses(RedisCache)] + } + ] + } + + async init(nodeData: INodeData, _: string, options: ICommonObject): Promise { + const credentialData = await getCredentialData(nodeData.credential ?? '', options) + const username = getCredentialParam('redisCacheUser', credentialData, nodeData) + const password = getCredentialParam('redisCachePwd', credentialData, nodeData) + const portStr = getCredentialParam('redisCachePort', credentialData, nodeData) + const host = getCredentialParam('redisCacheHost', credentialData, nodeData) + let port = 6379 + try { + port = portStr ? parseInt(portStr) : 6379 + } catch (e) { + port = 6379 + } + + const client = new Redis({ + port: port, // Redis port + host: host, + username: username, + password: password + }) + return new RedisCache(client) + } +} + +module.exports = { nodeClass: LLMRedisCache } diff --git a/packages/components/nodes/llmcache/LLMRedisCache/redis.svg b/packages/components/nodes/llmcache/LLMRedisCache/redis.svg new file mode 100644 index 00000000..90359069 --- /dev/null +++ b/packages/components/nodes/llmcache/LLMRedisCache/redis.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/packages/components/package.json b/packages/components/package.json index 18bf8e6a..5e993f86 100644 --- a/packages/components/package.json +++ b/packages/components/package.json @@ -43,6 +43,7 @@ "google-auth-library": "^9.0.0", "graphql": "^16.6.0", "html-to-text": "^9.0.5", + "ioredis": "^5.3.2", "langchain": "^0.0.154", "langfuse-langchain": "^1.0.14-alpha.0", "langsmith": "^0.0.32", From cddc68c187d6e430e1dce85497c8ff254caeb0a2 Mon Sep 17 00:00:00 2001 From: vinodkiran Date: Fri, 6 Oct 2023 13:17:56 +0530 Subject: [PATCH 06/25] LLM Cache - Addition of Upstash Redis --- .../credentials/UpstashRedisApi.credential.ts | 28 ++++++++ .../LLMUpstashRedisCache.ts | 66 ++++++++++++++++++ .../llmcache/LLMUpstashRedisCache/upstash.png | Bin 0 -> 2698 bytes packages/components/package.json | 3 +- 4 files changed, 96 insertions(+), 1 deletion(-) create mode 100644 packages/components/credentials/UpstashRedisApi.credential.ts create mode 100644 packages/components/nodes/llmcache/LLMUpstashRedisCache/LLMUpstashRedisCache.ts create mode 100644 packages/components/nodes/llmcache/LLMUpstashRedisCache/upstash.png diff --git a/packages/components/credentials/UpstashRedisApi.credential.ts b/packages/components/credentials/UpstashRedisApi.credential.ts new file mode 100644 index 00000000..b6e62ff3 --- /dev/null +++ b/packages/components/credentials/UpstashRedisApi.credential.ts @@ -0,0 +1,28 @@ +import { INodeParams, INodeCredential } from '../src/Interface' + +class UpstashRedisApi implements INodeCredential { + label: string + name: string + version: number + inputs: INodeParams[] + + constructor() { + this.label = 'Upstash Redis API' + this.name = 'upstashRedisApi' + this.version = 1.0 + this.inputs = [ + { + label: 'Upstash Redis REST URL', + name: 'upstashConnectionUrl', + type: 'string' + }, + { + label: 'Token', + name: 'upstashConnectionToken', + type: 'password' + } + ] + } +} + +module.exports = { credClass: UpstashRedisApi } diff --git a/packages/components/nodes/llmcache/LLMUpstashRedisCache/LLMUpstashRedisCache.ts b/packages/components/nodes/llmcache/LLMUpstashRedisCache/LLMUpstashRedisCache.ts new file mode 100644 index 00000000..b16d6e13 --- /dev/null +++ b/packages/components/nodes/llmcache/LLMUpstashRedisCache/LLMUpstashRedisCache.ts @@ -0,0 +1,66 @@ +import { + getBaseClasses, + getCredentialData, + getCredentialParam, + ICommonObject, + INode, + INodeData, + INodeOutputsValue, + INodeParams +} from '../../../src' +import { UpstashRedisCache } from 'langchain/cache/upstash_redis' + +class LLMUpstashRedisCache implements INode { + label: string + name: string + version: number + description: string + type: string + icon: string + category: string + baseClasses: string[] + inputs: INodeParams[] + outputs: INodeOutputsValue[] + credential: INodeParams + + constructor() { + this.label = 'Upstash Redis LLM Cache' + this.name = 'upstashRedisCache' + this.version = 1.0 + this.type = 'LLMCache' + this.icon = 'upstash.png' + this.category = 'LLM Cache' + this.baseClasses = [this.type, 'LLMCacheBase'] + this.credential = { + label: 'Connect Credential', + name: 'credential', + type: 'credential', + optional: true, + credentialNames: ['upstashRedisApi'] + } + this.inputs = [] + this.outputs = [ + { + label: 'LLM Cache', + name: 'cache', + baseClasses: [this.type, ...getBaseClasses(UpstashRedisCache)] + } + ] + } + + async init(nodeData: INodeData, _: string, options: ICommonObject): Promise { + const credentialData = await getCredentialData(nodeData.credential ?? '', options) + const upstashConnectionUrl = getCredentialParam('upstashConnectionUrl', credentialData, nodeData) + const upstashToken = getCredentialParam('upstashConnectionToken', credentialData, nodeData) + + const cache = new UpstashRedisCache({ + config: { + url: upstashConnectionUrl, + token: upstashToken + } + }) + return cache + } +} + +module.exports = { nodeClass: LLMUpstashRedisCache } diff --git a/packages/components/nodes/llmcache/LLMUpstashRedisCache/upstash.png b/packages/components/nodes/llmcache/LLMUpstashRedisCache/upstash.png new file mode 100644 index 0000000000000000000000000000000000000000..e27e02f4a3ba635cf04a33dbcfaa1a8e973ee2cc GIT binary patch literal 2698 zcmV;53U&2~P)Px#1ZP1_K>z@;j|==^1poj532;bRa{vGi!~g&e!~vBn4jTXf3Li;CK~z{rwODCv zRM#0jGhW6X+he@o1>0b-OTb{u1PC;ViGm<$NE4*0qHWrS0!gY?irP|KO4>->)$*rJ zgj#~s$`MLYnkFhK6bK=TB1l>j63k}sg24;M9`E+-i+jHN-psriKgT9Yj`cjA``)|v z+;2JGcWs#@@EM@fH-<}|K@0{xa0a{>341XX^1~YrBCJ1?3=1-oQjyJTPI4NGlkF%- zv7^>LSB=kvPfgYx4B+r+JKmq@MT>6)nbs7fTa%GwvGB6O%4aO{W6?g(&Pr)6NJuz|vhez7c6_}uPY_M696ge}Hrzy-9 zlreueq&y+yA8?dmYwl7MrrNa!h*^_8KGK1P!Di3^$f6a^2#S^D^2wr;L3sEX=65Tf zm1X27#hRo%kj71>^6zB&K)jJiLgsJ=-Kb8_!>)pr$S3I;$#(lE@bjM2aP#HtBpVlw zx0jHU*mQ>js7;%T($p+0;{ugw*_fA_sXrBiI}|{de;ijm!)Wu3qS-r$?tqK2o36Z+ zIDcR=gqz+md@XPBCzCzsY{tJv+f*_nCs;DF^HugjY;?@W{aGdYojZrP|L;Rr@!nWB z6ElE3t`{La5b&VFmW}7ZisO$ z+f8FAEwDMTBrO+}bnV)#dD=^}L>$>hS1*3q^FNd(SWVo%IeQVlnZHyYnTkg0?V+I- z^ia|y`)#}=&XdF^dKAe@l8}BTO^AO}BWQqPd@HX?$>2sZJvHpb`(iLS91 zVP~D6u%Dt7(6VG7K2s&)YEd3&0}Rr zL@TRLt9Jx8gX=@IXpryhE?$jld%iw8Jq|2-2(K=Gk_%B!xydM4$_KdNFI_prFf@!4 zimZptLr@FdD5Q?(vb>t1kCn?qHyTT$SanHyE^65B%@$kot8ldH8+5-%lZpr^n^&nW zPhUT-4`|Rl&+f#Mkg`HUB*+Z%s9Z6sf=OC>YGO#=@W|yuc%k=0eIy>6ax3vt#g}QZ zL4GsLm^KFuH{OMdMu|YNNwylrN`sJ@2>G-x32@L=(sW2NMIf=;>KQ?wxK+pN1I;Ko z`zLm6?f7BaVf?Q91N}}so+(;^n(T#|g^jT1VRQ3q`WKySX-Wpt7=NM)5v0B6QrLloakqm>+dOAja{sCOWDu!eYkwWFYw!<3_5LtGnSC*e5v3g|JbK1YyUZIQGy=a+s5& zMg%gR#YJB1Kaan%i=X1E1RP^y^4OJEIXkUwv?8T&E&GKv?28l8!Q=cpJSC{4jDU$~ zXJJ~aKOfg3M&Mz$w5{VU>aSHbDpu~+&M>)3N&1iTB5bTMPtC2t(*-NoOlPU{zf5y* z@Cwgu7w|C~6tKCDCrWQon#R`drl$HQo}wO#DIBn*xWwBVM1SZg*oiCUaxS7p1%lZ?p`M7kwzEDq95@&1L{XV^EGmt)JB z-)Qd}&vd?V)R*3mz2)oG*J;T{aFA8#NeVoqp-~z-O>!hcre$*kO50jTIZiEmLiMj^ z*vOE1d$?6A9udgqT~16s_yD#@&U>hNibN5NKo@$Z}b$F z`z8;4&$1`|Jn=C%`8ZwQ%|055|FvOxuX8NGd)4f96A&fa(BFh@SJ>gu!zFAwUgWO6 zl6RMS=(B>0WqBj5`2o8yMbemvLc>3bhq5bh!i1>6((iXo)-i#u@mv&jhr}K;F(bu} z?@*%tz--Jg>farrfwBG<_Op#F6kw>RcsD}goT^h_whnt)wx@$-d+U+OzjeOL!dx(Q z5|uEND0W{-T;vIs)^eWn*E{B;iV`HvhY`}>Ho5!p(fBRAKi-LREYVHsoFz)aXA?`Z z%ulyxQf1E7Y{%khDq93947z+{c(nPSxHv-jP&+q~7@0Uio+n$db<;W|Q3GCbPsoS| znC zg%Fg@G1DvPou8K0;i znH(~K9E0*%YRh=sDQ2cu+vcFq=D<3}dDSeA<}(BMKR;-3TtO!==>Px#07*qoM6N<$ Ef=GE79{>OV literal 0 HcmV?d00001 diff --git a/packages/components/package.json b/packages/components/package.json index 5e993f86..e3c4380a 100644 --- a/packages/components/package.json +++ b/packages/components/package.json @@ -29,6 +29,7 @@ "@supabase/supabase-js": "^2.29.0", "@types/js-yaml": "^4.0.5", "@types/jsdom": "^21.1.1", + "@upstash/redis": "^1.22.1", "@zilliz/milvus2-sdk-node": "^2.2.24", "apify-client": "^2.7.1", "axios": "^0.27.2", @@ -44,7 +45,7 @@ "graphql": "^16.6.0", "html-to-text": "^9.0.5", "ioredis": "^5.3.2", - "langchain": "^0.0.154", + "langchain": "^0.0.157", "langfuse-langchain": "^1.0.14-alpha.0", "langsmith": "^0.0.32", "linkifyjs": "^4.1.1", From 0b0cf357c58d8f88aaf22adc68abef41af607e80 Mon Sep 17 00:00:00 2001 From: Henry Heng Date: Fri, 6 Oct 2023 23:34:07 +0100 Subject: [PATCH 07/25] Update LLMInMemoryCache.ts code cleanup --- .../llmcache/LLMInMemoryCache/LLMInMemoryCache.ts | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/packages/components/nodes/llmcache/LLMInMemoryCache/LLMInMemoryCache.ts b/packages/components/nodes/llmcache/LLMInMemoryCache/LLMInMemoryCache.ts index 5e77a029..44133486 100644 --- a/packages/components/nodes/llmcache/LLMInMemoryCache/LLMInMemoryCache.ts +++ b/packages/components/nodes/llmcache/LLMInMemoryCache/LLMInMemoryCache.ts @@ -1,4 +1,4 @@ -import { INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface' +import { INode, INodeParams } from '../../../src/Interface' import { InMemoryCache } from 'langchain/cache' import { getBaseClasses } from '../../../src' @@ -12,7 +12,6 @@ class LLMInMemoryCache implements INode { category: string baseClasses: string[] inputs: INodeParams[] - outputs: INodeOutputsValue[] inMemoryCache: any constructor() { @@ -22,18 +21,11 @@ class LLMInMemoryCache implements INode { this.type = 'LLMCache' this.icon = 'memorycache.png' this.category = 'LLM Cache' - this.baseClasses = [this.type, 'LLMCacheBase'] + this.baseClasses = [this.type, ...getBaseClasses(InMemoryCache)] this.inputs = [] - this.outputs = [ - { - label: 'LLM Cache', - name: 'cache', - baseClasses: [this.type, ...getBaseClasses(InMemoryCache)] - } - ] } - async init(nodeData: INodeData): Promise { + async init(): Promise { if (!this.inMemoryCache) { this.inMemoryCache = InMemoryCache.global() } From 2785e3b685f6e5adcee00af3b52dcfec0f5af058 Mon Sep 17 00:00:00 2001 From: Henry Heng Date: Fri, 6 Oct 2023 23:34:36 +0100 Subject: [PATCH 08/25] Update LLMMomentoCache.ts code cleanup --- .../LLMMomentoCache/LLMMomentoCache.ts | 21 ++----------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/packages/components/nodes/llmcache/LLMMomentoCache/LLMMomentoCache.ts b/packages/components/nodes/llmcache/LLMMomentoCache/LLMMomentoCache.ts index 82267c24..f8fd61fb 100644 --- a/packages/components/nodes/llmcache/LLMMomentoCache/LLMMomentoCache.ts +++ b/packages/components/nodes/llmcache/LLMMomentoCache/LLMMomentoCache.ts @@ -1,13 +1,4 @@ -import { - getBaseClasses, - getCredentialData, - getCredentialParam, - ICommonObject, - INode, - INodeData, - INodeOutputsValue, - INodeParams -} from '../../../src' +import { getBaseClasses, getCredentialData, getCredentialParam, ICommonObject, INode, INodeData, INodeParams } from '../../../src' import { MomentoCache } from 'langchain/cache/momento' import { CacheClient, Configurations, CredentialProvider } from '@gomomento/sdk' @@ -21,7 +12,6 @@ class LLMMomentoCache implements INode { category: string baseClasses: string[] inputs: INodeParams[] - outputs: INodeOutputsValue[] credential: INodeParams constructor() { @@ -31,7 +21,7 @@ class LLMMomentoCache implements INode { this.type = 'LLMCache' this.icon = 'momento.png' this.category = 'LLM Cache' - this.baseClasses = [this.type, 'LLMCacheBase'] + this.baseClasses = [this.type, ...getBaseClasses(MomentoCache)] this.credential = { label: 'Connect Credential', name: 'credential', @@ -40,13 +30,6 @@ class LLMMomentoCache implements INode { credentialNames: ['momentoCacheApi'] } this.inputs = [] - this.outputs = [ - { - label: 'LLM Cache', - name: 'cache', - baseClasses: [this.type, ...getBaseClasses(MomentoCache)] - } - ] } async init(nodeData: INodeData, _: string, options: ICommonObject): Promise { From 52ccb89949cb75a32730bdd7fcaeb00fe8195cf6 Mon Sep 17 00:00:00 2001 From: Henry Heng Date: Fri, 6 Oct 2023 23:34:56 +0100 Subject: [PATCH 09/25] Update LLMRedisCache.ts code cleanup --- .../llmcache/LLMRedisCache/LLMRedisCache.ts | 35 ++++--------------- 1 file changed, 6 insertions(+), 29 deletions(-) diff --git a/packages/components/nodes/llmcache/LLMRedisCache/LLMRedisCache.ts b/packages/components/nodes/llmcache/LLMRedisCache/LLMRedisCache.ts index 4a098e82..a7208977 100644 --- a/packages/components/nodes/llmcache/LLMRedisCache/LLMRedisCache.ts +++ b/packages/components/nodes/llmcache/LLMRedisCache/LLMRedisCache.ts @@ -1,13 +1,4 @@ -import { - getBaseClasses, - getCredentialData, - getCredentialParam, - ICommonObject, - INode, - INodeData, - INodeOutputsValue, - INodeParams -} from '../../../src' +import { getBaseClasses, getCredentialData, getCredentialParam, ICommonObject, INode, INodeData, INodeParams } from '../../../src' import { RedisCache } from 'langchain/cache/ioredis' import { Redis } from 'ioredis' @@ -21,7 +12,6 @@ class LLMRedisCache implements INode { category: string baseClasses: string[] inputs: INodeParams[] - outputs: INodeOutputsValue[] credential: INodeParams constructor() { @@ -31,7 +21,7 @@ class LLMRedisCache implements INode { this.type = 'LLMCache' this.icon = 'redis.svg' this.category = 'LLM Cache' - this.baseClasses = [this.type, 'LLMCacheBase'] + this.baseClasses = [this.type, ...getBaseClasses(RedisCache)] this.credential = { label: 'Connect Credential', name: 'credential', @@ -40,13 +30,6 @@ class LLMRedisCache implements INode { credentialNames: ['redisCacheApi'] } this.inputs = [] - this.outputs = [ - { - label: 'LLM Cache', - name: 'cache', - baseClasses: [this.type, ...getBaseClasses(RedisCache)] - } - ] } async init(nodeData: INodeData, _: string, options: ICommonObject): Promise { @@ -55,18 +38,12 @@ class LLMRedisCache implements INode { const password = getCredentialParam('redisCachePwd', credentialData, nodeData) const portStr = getCredentialParam('redisCachePort', credentialData, nodeData) const host = getCredentialParam('redisCacheHost', credentialData, nodeData) - let port = 6379 - try { - port = portStr ? parseInt(portStr) : 6379 - } catch (e) { - port = 6379 - } const client = new Redis({ - port: port, // Redis port - host: host, - username: username, - password: password + port: portStr ? parseInt(portStr) : 6379, + host, + username, + password }) return new RedisCache(client) } From 5288ea8155447fb75b843eec05deb540231d249d Mon Sep 17 00:00:00 2001 From: Henry Heng Date: Fri, 6 Oct 2023 23:35:16 +0100 Subject: [PATCH 10/25] Update LLMUpstashRedisCache.ts code cleanup --- .../LLMUpstashRedisCache.ts | 21 ++----------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/packages/components/nodes/llmcache/LLMUpstashRedisCache/LLMUpstashRedisCache.ts b/packages/components/nodes/llmcache/LLMUpstashRedisCache/LLMUpstashRedisCache.ts index b16d6e13..8914b2a1 100644 --- a/packages/components/nodes/llmcache/LLMUpstashRedisCache/LLMUpstashRedisCache.ts +++ b/packages/components/nodes/llmcache/LLMUpstashRedisCache/LLMUpstashRedisCache.ts @@ -1,13 +1,4 @@ -import { - getBaseClasses, - getCredentialData, - getCredentialParam, - ICommonObject, - INode, - INodeData, - INodeOutputsValue, - INodeParams -} from '../../../src' +import { getBaseClasses, getCredentialData, getCredentialParam, ICommonObject, INode, INodeData, INodeParams } from '../../../src' import { UpstashRedisCache } from 'langchain/cache/upstash_redis' class LLMUpstashRedisCache implements INode { @@ -20,7 +11,6 @@ class LLMUpstashRedisCache implements INode { category: string baseClasses: string[] inputs: INodeParams[] - outputs: INodeOutputsValue[] credential: INodeParams constructor() { @@ -30,7 +20,7 @@ class LLMUpstashRedisCache implements INode { this.type = 'LLMCache' this.icon = 'upstash.png' this.category = 'LLM Cache' - this.baseClasses = [this.type, 'LLMCacheBase'] + this.baseClasses = [this.type, ...getBaseClasses(UpstashRedisCache)] this.credential = { label: 'Connect Credential', name: 'credential', @@ -39,13 +29,6 @@ class LLMUpstashRedisCache implements INode { credentialNames: ['upstashRedisApi'] } this.inputs = [] - this.outputs = [ - { - label: 'LLM Cache', - name: 'cache', - baseClasses: [this.type, ...getBaseClasses(UpstashRedisCache)] - } - ] } async init(nodeData: INodeData, _: string, options: ICommonObject): Promise { From 8ee4f0fc3c71971d7408c65d91d9b9f3f7735c46 Mon Sep 17 00:00:00 2001 From: Henry Heng Date: Fri, 6 Oct 2023 23:36:09 +0100 Subject: [PATCH 11/25] Update Interface.ts code cleanup --- packages/components/src/Interface.ts | 7 ------- 1 file changed, 7 deletions(-) diff --git a/packages/components/src/Interface.ts b/packages/components/src/Interface.ts index 76dc7354..e883d056 100644 --- a/packages/components/src/Interface.ts +++ b/packages/components/src/Interface.ts @@ -1,7 +1,6 @@ /** * Types */ -import { BaseCache } from 'langchain/schema' export type NodeParamsType = | 'asyncOptions' @@ -177,9 +176,3 @@ export class VectorStoreRetriever { this.vectorStore = fields.vectorStore } } - -export interface LLMCacheBase { - name: string - description: string - baseCache: BaseCache -} From dbd655580d9bf68c7976b0b4502426ad8774ff3c Mon Sep 17 00:00:00 2001 From: Henry Heng Date: Sat, 7 Oct 2023 19:17:33 +0100 Subject: [PATCH 12/25] Add fix where flow is not restarted when cache is used --- packages/server/src/utils/index.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/server/src/utils/index.ts b/packages/server/src/utils/index.ts index bb048490..98802cde 100644 --- a/packages/server/src/utils/index.ts +++ b/packages/server/src/utils/index.ts @@ -477,6 +477,7 @@ export const replaceInputsWithConfig = (flowNodeData: INodeData, overrideConfig: */ export const isStartNodeDependOnInput = (startingNodes: IReactFlowNode[], nodes: IReactFlowNode[]): boolean => { for (const node of startingNodes) { + if (node.data.category === 'LLM Cache') return true for (const inputName in node.data.inputs) { const inputVariables = getInputVariables(node.data.inputs[inputName]) if (inputVariables.length > 0) return true From 679eac1d0b24c5b7b8ba6a582dcf88ebc4589588 Mon Sep 17 00:00:00 2001 From: vinodkiran Date: Sun, 8 Oct 2023 11:44:43 +0530 Subject: [PATCH 13/25] Addition of Cache Option for Chat Models --- .../nodes/chatmodels/AWSBedrock/AWSChatBedrock.ts | 14 ++++++++++++-- .../chatmodels/AzureChatOpenAI/AzureChatOpenAI.ts | 14 ++++++++++++-- .../nodes/chatmodels/Bittensor/Bittensor.ts | 13 ++++++++++++- .../chatmodels/ChatAnthropic/ChatAnthropic.ts | 14 ++++++++++++-- .../chatmodels/ChatGooglePaLM/ChatGooglePaLM.ts | 11 ++++++++++- .../ChatGoogleVertexAI/ChatGoogleVertexAI.ts | 11 ++++++++++- .../chatmodels/ChatHuggingFace/ChatHuggingFace.ts | 11 ++++++++++- .../nodes/chatmodels/ChatLocalAI/ChatLocalAI.ts | 14 ++++++++++++-- .../nodes/chatmodels/ChatOpenAI/ChatOpenAI.ts | 15 +++++++++++++-- .../ChatOpenAICustom/ChatOpenAICustom.ts | 14 ++++++++++++-- 10 files changed, 115 insertions(+), 16 deletions(-) diff --git a/packages/components/nodes/chatmodels/AWSBedrock/AWSChatBedrock.ts b/packages/components/nodes/chatmodels/AWSBedrock/AWSChatBedrock.ts index f5fa8bba..ba4bdd8a 100644 --- a/packages/components/nodes/chatmodels/AWSBedrock/AWSChatBedrock.ts +++ b/packages/components/nodes/chatmodels/AWSBedrock/AWSChatBedrock.ts @@ -2,6 +2,8 @@ import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Inter import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils' import { ChatBedrock } from 'langchain/chat_models/bedrock' import { BaseBedrockInput } from 'langchain/dist/util/bedrock' +import { BaseCache } from 'langchain/schema' +import { BaseLLMParams } from 'langchain/llms/base' /** * I had to run the following to build the component @@ -25,7 +27,7 @@ class AWSChatBedrock_ChatModels implements INode { constructor() { this.label = 'AWS Bedrock' this.name = 'awsChatBedrock' - this.version = 1.1 + this.version = 2.0 this.type = 'AWSChatBedrock' this.icon = 'awsBedrock.png' this.category = 'Chat Models' @@ -39,6 +41,12 @@ class AWSChatBedrock_ChatModels implements INode { optional: true } this.inputs = [ + { + label: 'Cache', + name: 'cache', + type: 'LLMCache', + optional: true + }, { label: 'Region', name: 'region', @@ -130,8 +138,9 @@ class AWSChatBedrock_ChatModels implements INode { const iModel = nodeData.inputs?.model as string const iTemperature = nodeData.inputs?.temperature as string const iMax_tokens_to_sample = nodeData.inputs?.max_tokens_to_sample as string + const cache = nodeData.inputs?.llmCache as BaseCache - const obj: BaseBedrockInput = { + const obj: BaseBedrockInput & BaseLLMParams = { region: iRegion, model: iModel, maxTokens: parseInt(iMax_tokens_to_sample, 10), @@ -157,6 +166,7 @@ class AWSChatBedrock_ChatModels implements INode { sessionToken: credentialApiSession } } + if (cache) obj.cache = cache const amazonBedrock = new ChatBedrock(obj) return amazonBedrock diff --git a/packages/components/nodes/chatmodels/AzureChatOpenAI/AzureChatOpenAI.ts b/packages/components/nodes/chatmodels/AzureChatOpenAI/AzureChatOpenAI.ts index 90f430f0..5acf4dce 100644 --- a/packages/components/nodes/chatmodels/AzureChatOpenAI/AzureChatOpenAI.ts +++ b/packages/components/nodes/chatmodels/AzureChatOpenAI/AzureChatOpenAI.ts @@ -2,6 +2,8 @@ import { OpenAIBaseInput } from 'langchain/dist/types/openai-types' import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface' import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils' import { AzureOpenAIInput, ChatOpenAI } from 'langchain/chat_models/openai' +import { BaseCache } from 'langchain/schema' +import { BaseLLMParams } from 'langchain/llms/base' class AzureChatOpenAI_ChatModels implements INode { label: string @@ -18,7 +20,7 @@ class AzureChatOpenAI_ChatModels implements INode { constructor() { this.label = 'Azure ChatOpenAI' this.name = 'azureChatOpenAI' - this.version = 1.0 + this.version = 2.0 this.type = 'AzureChatOpenAI' this.icon = 'Azure.svg' this.category = 'Chat Models' @@ -31,6 +33,12 @@ class AzureChatOpenAI_ChatModels implements INode { credentialNames: ['azureOpenAIApi'] } this.inputs = [ + { + label: 'Cache', + name: 'cache', + type: 'LLMCache', + optional: true + }, { label: 'Model Name', name: 'modelName', @@ -107,6 +115,7 @@ class AzureChatOpenAI_ChatModels implements INode { const presencePenalty = nodeData.inputs?.presencePenalty as string const timeout = nodeData.inputs?.timeout as string const streaming = nodeData.inputs?.streaming as boolean + const cache = nodeData.inputs?.llmCache as BaseCache const credentialData = await getCredentialData(nodeData.credential ?? '', options) const azureOpenAIApiKey = getCredentialParam('azureOpenAIApiKey', credentialData, nodeData) @@ -114,7 +123,7 @@ class AzureChatOpenAI_ChatModels implements INode { const azureOpenAIApiDeploymentName = getCredentialParam('azureOpenAIApiDeploymentName', credentialData, nodeData) const azureOpenAIApiVersion = getCredentialParam('azureOpenAIApiVersion', credentialData, nodeData) - const obj: Partial & Partial = { + const obj: Partial & BaseLLMParams & Partial = { temperature: parseFloat(temperature), modelName, azureOpenAIApiKey, @@ -128,6 +137,7 @@ class AzureChatOpenAI_ChatModels implements INode { if (frequencyPenalty) obj.frequencyPenalty = parseFloat(frequencyPenalty) if (presencePenalty) obj.presencePenalty = parseFloat(presencePenalty) if (timeout) obj.timeout = parseInt(timeout, 10) + if (cache) obj.cache = cache const model = new ChatOpenAI(obj) return model diff --git a/packages/components/nodes/chatmodels/Bittensor/Bittensor.ts b/packages/components/nodes/chatmodels/Bittensor/Bittensor.ts index cfcac863..89c37157 100644 --- a/packages/components/nodes/chatmodels/Bittensor/Bittensor.ts +++ b/packages/components/nodes/chatmodels/Bittensor/Bittensor.ts @@ -1,6 +1,7 @@ import { INode, INodeData, INodeParams } from '../../../src/Interface' import { getBaseClasses } from '../../../src/utils' import { NIBittensorChatModel, BittensorInput } from 'langchain/experimental/chat_models/bittensor' +import { BaseCache } from 'langchain/schema' class Bittensor_ChatModels implements INode { label: string @@ -16,13 +17,19 @@ class Bittensor_ChatModels implements INode { constructor() { this.label = 'NIBittensorChat' this.name = 'NIBittensorChatModel' - this.version = 1.0 + this.version = 2.0 this.type = 'BittensorChat' this.icon = 'logo.png' this.category = 'Chat Models' this.description = 'Wrapper around Bittensor subnet 1 large language models' this.baseClasses = [this.type, ...getBaseClasses(NIBittensorChatModel)] this.inputs = [ + { + label: 'Cache', + name: 'cache', + type: 'LLMCache', + optional: true + }, { label: 'System prompt', name: 'system_prompt', @@ -35,9 +42,13 @@ class Bittensor_ChatModels implements INode { async init(nodeData: INodeData, _: string): Promise { const system_prompt = nodeData.inputs?.system_prompt as string + const cache = nodeData.inputs?.llmCache as BaseCache + const obj: Partial = { systemPrompt: system_prompt } + if (cache) obj.cache = cache + const model = new NIBittensorChatModel(obj) return model } diff --git a/packages/components/nodes/chatmodels/ChatAnthropic/ChatAnthropic.ts b/packages/components/nodes/chatmodels/ChatAnthropic/ChatAnthropic.ts index 12a33d99..039b437b 100644 --- a/packages/components/nodes/chatmodels/ChatAnthropic/ChatAnthropic.ts +++ b/packages/components/nodes/chatmodels/ChatAnthropic/ChatAnthropic.ts @@ -1,6 +1,8 @@ import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface' import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils' import { AnthropicInput, ChatAnthropic } from 'langchain/chat_models/anthropic' +import { BaseCache } from 'langchain/schema' +import { BaseLLMParams } from 'langchain/llms/base' class ChatAnthropic_ChatModels implements INode { label: string @@ -17,7 +19,7 @@ class ChatAnthropic_ChatModels implements INode { constructor() { this.label = 'ChatAnthropic' this.name = 'chatAnthropic' - this.version = 1.0 + this.version = 2.0 this.type = 'ChatAnthropic' this.icon = 'chatAnthropic.png' this.category = 'Chat Models' @@ -30,6 +32,12 @@ class ChatAnthropic_ChatModels implements INode { credentialNames: ['anthropicApi'] } this.inputs = [ + { + label: 'Cache', + name: 'cache', + type: 'LLMCache', + optional: true + }, { label: 'Model Name', name: 'modelName', @@ -135,11 +143,12 @@ class ChatAnthropic_ChatModels implements INode { const topP = nodeData.inputs?.topP as string const topK = nodeData.inputs?.topK as string const streaming = nodeData.inputs?.streaming as boolean + const cache = nodeData.inputs?.llmCache as BaseCache const credentialData = await getCredentialData(nodeData.credential ?? '', options) const anthropicApiKey = getCredentialParam('anthropicApiKey', credentialData, nodeData) - const obj: Partial & { anthropicApiKey?: string } = { + const obj: Partial & BaseLLMParams & { anthropicApiKey?: string } = { temperature: parseFloat(temperature), modelName, anthropicApiKey, @@ -149,6 +158,7 @@ class ChatAnthropic_ChatModels implements INode { if (maxTokensToSample) obj.maxTokensToSample = parseInt(maxTokensToSample, 10) if (topP) obj.topP = parseFloat(topP) if (topK) obj.topK = parseFloat(topK) + if (cache) obj.cache = cache const model = new ChatAnthropic(obj) return model diff --git a/packages/components/nodes/chatmodels/ChatGooglePaLM/ChatGooglePaLM.ts b/packages/components/nodes/chatmodels/ChatGooglePaLM/ChatGooglePaLM.ts index 642ddb5e..8ee3fbb8 100644 --- a/packages/components/nodes/chatmodels/ChatGooglePaLM/ChatGooglePaLM.ts +++ b/packages/components/nodes/chatmodels/ChatGooglePaLM/ChatGooglePaLM.ts @@ -1,6 +1,7 @@ import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface' import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils' import { ChatGooglePaLM, GooglePaLMChatInput } from 'langchain/chat_models/googlepalm' +import { BaseCache } from 'langchain/schema' class ChatGooglePaLM_ChatModels implements INode { label: string @@ -17,7 +18,7 @@ class ChatGooglePaLM_ChatModels implements INode { constructor() { this.label = 'ChatGooglePaLM' this.name = 'chatGooglePaLM' - this.version = 1.0 + this.version = 2.0 this.type = 'ChatGooglePaLM' this.icon = 'Google_PaLM_Logo.svg' this.category = 'Chat Models' @@ -30,6 +31,12 @@ class ChatGooglePaLM_ChatModels implements INode { credentialNames: ['googleMakerSuite'] } this.inputs = [ + { + label: 'Cache', + name: 'cache', + type: 'LLMCache', + optional: true + }, { label: 'Model Name', name: 'modelName', @@ -96,6 +103,7 @@ class ChatGooglePaLM_ChatModels implements INode { const temperature = nodeData.inputs?.temperature as string const topP = nodeData.inputs?.topP as string const topK = nodeData.inputs?.topK as string + const cache = nodeData.inputs?.llmCache as BaseCache const credentialData = await getCredentialData(nodeData.credential ?? '', options) const googleMakerSuiteKey = getCredentialParam('googleMakerSuiteKey', credentialData, nodeData) @@ -108,6 +116,7 @@ class ChatGooglePaLM_ChatModels implements INode { if (topP) obj.topP = parseFloat(topP) if (topK) obj.topK = parseFloat(topK) + if (cache) obj.cache = cache const model = new ChatGooglePaLM(obj) return model diff --git a/packages/components/nodes/chatmodels/ChatGoogleVertexAI/ChatGoogleVertexAI.ts b/packages/components/nodes/chatmodels/ChatGoogleVertexAI/ChatGoogleVertexAI.ts index 4cb206f5..4f876239 100644 --- a/packages/components/nodes/chatmodels/ChatGoogleVertexAI/ChatGoogleVertexAI.ts +++ b/packages/components/nodes/chatmodels/ChatGoogleVertexAI/ChatGoogleVertexAI.ts @@ -2,6 +2,7 @@ import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Inter import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils' import { ChatGoogleVertexAI, GoogleVertexAIChatInput } from 'langchain/chat_models/googlevertexai' import { GoogleAuthOptions } from 'google-auth-library' +import { BaseCache } from 'langchain/schema' class GoogleVertexAI_ChatModels implements INode { label: string @@ -18,7 +19,7 @@ class GoogleVertexAI_ChatModels implements INode { constructor() { this.label = 'ChatGoogleVertexAI' this.name = 'chatGoogleVertexAI' - this.version = 1.0 + this.version = 2.0 this.type = 'ChatGoogleVertexAI' this.icon = 'vertexai.svg' this.category = 'Chat Models' @@ -34,6 +35,12 @@ class GoogleVertexAI_ChatModels implements INode { 'Google Vertex AI credential. If you are using a GCP service like Cloud Run, or if you have installed default credentials on your local machine, you do not need to set this credential.' } this.inputs = [ + { + label: 'Cache', + name: 'cache', + type: 'LLMCache', + optional: true + }, { label: 'Model Name', name: 'modelName', @@ -113,6 +120,7 @@ class GoogleVertexAI_ChatModels implements INode { const modelName = nodeData.inputs?.modelName as string const maxOutputTokens = nodeData.inputs?.maxOutputTokens as string const topP = nodeData.inputs?.topP as string + const cache = nodeData.inputs?.llmCache as BaseCache const obj: GoogleVertexAIChatInput = { temperature: parseFloat(temperature), @@ -122,6 +130,7 @@ class GoogleVertexAI_ChatModels implements INode { if (maxOutputTokens) obj.maxOutputTokens = parseInt(maxOutputTokens, 10) if (topP) obj.topP = parseFloat(topP) + if (cache) obj.cache = cache const model = new ChatGoogleVertexAI(obj) return model diff --git a/packages/components/nodes/chatmodels/ChatHuggingFace/ChatHuggingFace.ts b/packages/components/nodes/chatmodels/ChatHuggingFace/ChatHuggingFace.ts index ee55c7bb..d9c0d22f 100644 --- a/packages/components/nodes/chatmodels/ChatHuggingFace/ChatHuggingFace.ts +++ b/packages/components/nodes/chatmodels/ChatHuggingFace/ChatHuggingFace.ts @@ -1,6 +1,7 @@ import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface' import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils' import { HFInput, HuggingFaceInference } from './core' +import { BaseCache } from 'langchain/schema' class ChatHuggingFace_ChatModels implements INode { label: string @@ -17,7 +18,7 @@ class ChatHuggingFace_ChatModels implements INode { constructor() { this.label = 'ChatHuggingFace' this.name = 'chatHuggingFace' - this.version = 1.0 + this.version = 2.0 this.type = 'ChatHuggingFace' this.icon = 'huggingface.png' this.category = 'Chat Models' @@ -30,6 +31,12 @@ class ChatHuggingFace_ChatModels implements INode { credentialNames: ['huggingFaceApi'] } this.inputs = [ + { + label: 'Cache', + name: 'cache', + type: 'LLMCache', + optional: true + }, { label: 'Model', name: 'model', @@ -102,6 +109,7 @@ class ChatHuggingFace_ChatModels implements INode { const hfTopK = nodeData.inputs?.hfTopK as string const frequencyPenalty = nodeData.inputs?.frequencyPenalty as string const endpoint = nodeData.inputs?.endpoint as string + const cache = nodeData.inputs?.llmCache as BaseCache const credentialData = await getCredentialData(nodeData.credential ?? '', options) const huggingFaceApiKey = getCredentialParam('huggingFaceApiKey', credentialData, nodeData) @@ -119,6 +127,7 @@ class ChatHuggingFace_ChatModels implements INode { if (endpoint) obj.endpoint = endpoint const huggingFace = new HuggingFaceInference(obj) + if (cache) huggingFace.cache = cache return huggingFace } } diff --git a/packages/components/nodes/chatmodels/ChatLocalAI/ChatLocalAI.ts b/packages/components/nodes/chatmodels/ChatLocalAI/ChatLocalAI.ts index a6ddfae4..78fe31dd 100644 --- a/packages/components/nodes/chatmodels/ChatLocalAI/ChatLocalAI.ts +++ b/packages/components/nodes/chatmodels/ChatLocalAI/ChatLocalAI.ts @@ -2,6 +2,8 @@ import { INode, INodeData, INodeParams } from '../../../src/Interface' import { getBaseClasses } from '../../../src/utils' import { OpenAIChat } from 'langchain/llms/openai' import { OpenAIChatInput } from 'langchain/chat_models/openai' +import { BaseCache } from 'langchain/schema' +import { BaseLLMParams } from 'langchain/llms/base' class ChatLocalAI_ChatModels implements INode { label: string @@ -17,13 +19,19 @@ class ChatLocalAI_ChatModels implements INode { constructor() { this.label = 'ChatLocalAI' this.name = 'chatLocalAI' - this.version = 1.0 + this.version = 2.0 this.type = 'ChatLocalAI' this.icon = 'localai.png' this.category = 'Chat Models' this.description = 'Use local LLMs like llama.cpp, gpt4all using LocalAI' this.baseClasses = [this.type, 'BaseChatModel', ...getBaseClasses(OpenAIChat)] this.inputs = [ + { + label: 'Cache', + name: 'cache', + type: 'LLMCache', + optional: true + }, { label: 'Base Path', name: 'basePath', @@ -78,8 +86,9 @@ class ChatLocalAI_ChatModels implements INode { const topP = nodeData.inputs?.topP as string const timeout = nodeData.inputs?.timeout as string const basePath = nodeData.inputs?.basePath as string + const cache = nodeData.inputs?.llmCache as BaseCache - const obj: Partial & { openAIApiKey?: string } = { + const obj: Partial & BaseLLMParams & { openAIApiKey?: string } = { temperature: parseFloat(temperature), modelName, openAIApiKey: 'sk-' @@ -88,6 +97,7 @@ class ChatLocalAI_ChatModels implements INode { if (maxTokens) obj.maxTokens = parseInt(maxTokens, 10) if (topP) obj.topP = parseFloat(topP) if (timeout) obj.timeout = parseInt(timeout, 10) + if (cache) obj.cache = cache const model = new OpenAIChat(obj, { basePath }) diff --git a/packages/components/nodes/chatmodels/ChatOpenAI/ChatOpenAI.ts b/packages/components/nodes/chatmodels/ChatOpenAI/ChatOpenAI.ts index ca081ff4..a9eae481 100644 --- a/packages/components/nodes/chatmodels/ChatOpenAI/ChatOpenAI.ts +++ b/packages/components/nodes/chatmodels/ChatOpenAI/ChatOpenAI.ts @@ -1,6 +1,8 @@ import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface' import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils' import { ChatOpenAI, OpenAIChatInput } from 'langchain/chat_models/openai' +import { BaseCache } from 'langchain/schema' +import { BaseLLMParams } from 'langchain/llms/base' class ChatOpenAI_ChatModels implements INode { label: string @@ -17,7 +19,7 @@ class ChatOpenAI_ChatModels implements INode { constructor() { this.label = 'ChatOpenAI' this.name = 'chatOpenAI' - this.version = 1.0 + this.version = 2.0 this.type = 'ChatOpenAI' this.icon = 'openai.png' this.category = 'Chat Models' @@ -30,6 +32,12 @@ class ChatOpenAI_ChatModels implements INode { credentialNames: ['openAIApi'] } this.inputs = [ + { + label: 'Cache', + name: 'cache', + type: 'LLMCache', + optional: true + }, { label: 'Model Name', name: 'modelName', @@ -151,7 +159,9 @@ class ChatOpenAI_ChatModels implements INode { const credentialData = await getCredentialData(nodeData.credential ?? '', options) const openAIApiKey = getCredentialParam('openAIApiKey', credentialData, nodeData) - const obj: Partial & { openAIApiKey?: string } = { + const cache = nodeData.inputs?.llmCache as BaseCache + + const obj: Partial & BaseLLMParams & { openAIApiKey?: string } = { temperature: parseFloat(temperature), modelName, openAIApiKey, @@ -163,6 +173,7 @@ class ChatOpenAI_ChatModels implements INode { if (frequencyPenalty) obj.frequencyPenalty = parseFloat(frequencyPenalty) if (presencePenalty) obj.presencePenalty = parseFloat(presencePenalty) if (timeout) obj.timeout = parseInt(timeout, 10) + if (cache) obj.cache = cache let parsedBaseOptions: any | undefined = undefined diff --git a/packages/components/nodes/chatmodels/ChatOpenAICustom/ChatOpenAICustom.ts b/packages/components/nodes/chatmodels/ChatOpenAICustom/ChatOpenAICustom.ts index 29c1181a..6f8d1310 100644 --- a/packages/components/nodes/chatmodels/ChatOpenAICustom/ChatOpenAICustom.ts +++ b/packages/components/nodes/chatmodels/ChatOpenAICustom/ChatOpenAICustom.ts @@ -1,6 +1,8 @@ import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface' import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils' import { ChatOpenAI, OpenAIChatInput } from 'langchain/chat_models/openai' +import { BaseCache } from 'langchain/schema' +import { BaseLLMParams } from 'langchain/llms/base' class ChatOpenAICustom_ChatModels implements INode { label: string @@ -17,7 +19,7 @@ class ChatOpenAICustom_ChatModels implements INode { constructor() { this.label = 'ChatOpenAI Custom' this.name = 'chatOpenAICustom' - this.version = 1.0 + this.version = 2.0 this.type = 'ChatOpenAI-Custom' this.icon = 'openai.png' this.category = 'Chat Models' @@ -31,6 +33,12 @@ class ChatOpenAICustom_ChatModels implements INode { optional: true } this.inputs = [ + { + label: 'Cache', + name: 'cache', + type: 'LLMCache', + optional: true + }, { label: 'Model Name', name: 'modelName', @@ -113,11 +121,12 @@ class ChatOpenAICustom_ChatModels implements INode { const streaming = nodeData.inputs?.streaming as boolean const basePath = nodeData.inputs?.basepath as string const baseOptions = nodeData.inputs?.baseOptions + const cache = nodeData.inputs?.llmCache as BaseCache const credentialData = await getCredentialData(nodeData.credential ?? '', options) const openAIApiKey = getCredentialParam('openAIApiKey', credentialData, nodeData) - const obj: Partial & { openAIApiKey?: string } = { + const obj: Partial & BaseLLMParams & { openAIApiKey?: string } = { temperature: parseFloat(temperature), modelName, openAIApiKey, @@ -129,6 +138,7 @@ class ChatOpenAICustom_ChatModels implements INode { if (frequencyPenalty) obj.frequencyPenalty = parseFloat(frequencyPenalty) if (presencePenalty) obj.presencePenalty = parseFloat(presencePenalty) if (timeout) obj.timeout = parseInt(timeout, 10) + if (cache) obj.cache = cache let parsedBaseOptions: any | undefined = undefined From e01acbc3e8f1bf866e168890066c9183afefaf25 Mon Sep 17 00:00:00 2001 From: vinodkiran Date: Sun, 8 Oct 2023 15:53:06 +0530 Subject: [PATCH 14/25] Renaming of Category, Class Names and File names from LLMCache to Cache as this functionality is applicable to both LLM and Chat Models. --- .../InMemoryCache/InMemoryCache.ts} | 12 ++++++------ .../InMemoryCache}/memorycache.png | Bin .../MomentoCache/MomentoCache.ts} | 12 ++++++------ .../MomentoCache}/momento.png | Bin .../RedisCache/RedisCache.ts} | 12 ++++++------ .../LLMRedisCache => cache/RedisCache}/redis.svg | 0 .../UpstashRedisCache/UpstashRedisCache.ts} | 12 ++++++------ .../UpstashRedisCache}/upstash.png | Bin 8 files changed, 24 insertions(+), 24 deletions(-) rename packages/components/nodes/{llmcache/LLMInMemoryCache/LLMInMemoryCache.ts => cache/InMemoryCache/InMemoryCache.ts} (71%) rename packages/components/nodes/{llmcache/LLMInMemoryCache => cache/InMemoryCache}/memorycache.png (100%) rename packages/components/nodes/{llmcache/LLMMomentoCache/LLMMomentoCache.ts => cache/MomentoCache/MomentoCache.ts} (86%) rename packages/components/nodes/{llmcache/LLMMomentoCache => cache/MomentoCache}/momento.png (100%) rename packages/components/nodes/{llmcache/LLMRedisCache/LLMRedisCache.ts => cache/RedisCache/RedisCache.ts} (85%) rename packages/components/nodes/{llmcache/LLMRedisCache => cache/RedisCache}/redis.svg (100%) rename packages/components/nodes/{llmcache/LLMUpstashRedisCache/LLMUpstashRedisCache.ts => cache/UpstashRedisCache/UpstashRedisCache.ts} (82%) rename packages/components/nodes/{llmcache/LLMUpstashRedisCache => cache/UpstashRedisCache}/upstash.png (100%) diff --git a/packages/components/nodes/llmcache/LLMInMemoryCache/LLMInMemoryCache.ts b/packages/components/nodes/cache/InMemoryCache/InMemoryCache.ts similarity index 71% rename from packages/components/nodes/llmcache/LLMInMemoryCache/LLMInMemoryCache.ts rename to packages/components/nodes/cache/InMemoryCache/InMemoryCache.ts index 44133486..8bed721f 100644 --- a/packages/components/nodes/llmcache/LLMInMemoryCache/LLMInMemoryCache.ts +++ b/packages/components/nodes/cache/InMemoryCache/InMemoryCache.ts @@ -1,8 +1,8 @@ import { INode, INodeParams } from '../../../src/Interface' -import { InMemoryCache } from 'langchain/cache' +import { InMemoryCache as LangchainInMemoryCache } from 'langchain/cache' import { getBaseClasses } from '../../../src' -class LLMInMemoryCache implements INode { +class InMemoryCache implements INode { label: string name: string version: number @@ -18,19 +18,19 @@ class LLMInMemoryCache implements INode { this.label = 'Local (Builtin) Cache' this.name = 'localCache' this.version = 1.0 - this.type = 'LLMCache' + this.type = 'Cache' this.icon = 'memorycache.png' - this.category = 'LLM Cache' + this.category = 'Cache' this.baseClasses = [this.type, ...getBaseClasses(InMemoryCache)] this.inputs = [] } async init(): Promise { if (!this.inMemoryCache) { - this.inMemoryCache = InMemoryCache.global() + this.inMemoryCache = LangchainInMemoryCache.global() } return this.inMemoryCache } } -module.exports = { nodeClass: LLMInMemoryCache } +module.exports = { nodeClass: InMemoryCache } diff --git a/packages/components/nodes/llmcache/LLMInMemoryCache/memorycache.png b/packages/components/nodes/cache/InMemoryCache/memorycache.png similarity index 100% rename from packages/components/nodes/llmcache/LLMInMemoryCache/memorycache.png rename to packages/components/nodes/cache/InMemoryCache/memorycache.png diff --git a/packages/components/nodes/llmcache/LLMMomentoCache/LLMMomentoCache.ts b/packages/components/nodes/cache/MomentoCache/MomentoCache.ts similarity index 86% rename from packages/components/nodes/llmcache/LLMMomentoCache/LLMMomentoCache.ts rename to packages/components/nodes/cache/MomentoCache/MomentoCache.ts index f8fd61fb..37d90941 100644 --- a/packages/components/nodes/llmcache/LLMMomentoCache/LLMMomentoCache.ts +++ b/packages/components/nodes/cache/MomentoCache/MomentoCache.ts @@ -1,8 +1,8 @@ import { getBaseClasses, getCredentialData, getCredentialParam, ICommonObject, INode, INodeData, INodeParams } from '../../../src' -import { MomentoCache } from 'langchain/cache/momento' +import { MomentoCache as LangchainMomentoCache } from 'langchain/cache/momento' import { CacheClient, Configurations, CredentialProvider } from '@gomomento/sdk' -class LLMMomentoCache implements INode { +class MomentoCache implements INode { label: string name: string version: number @@ -18,9 +18,9 @@ class LLMMomentoCache implements INode { this.label = 'Momento Cache' this.name = 'momentoCache' this.version = 1.0 - this.type = 'LLMCache' + this.type = 'Cache' this.icon = 'momento.png' - this.category = 'LLM Cache' + this.category = 'Cache' this.baseClasses = [this.type, ...getBaseClasses(MomentoCache)] this.credential = { label: 'Connect Credential', @@ -47,7 +47,7 @@ class LLMMomentoCache implements INode { defaultTtlSeconds: 60 * 60 * 24 }) - let momentoCache = await MomentoCache.fromProps({ + let momentoCache = await LangchainMomentoCache.fromProps({ client, cacheName: cacheName }) @@ -55,4 +55,4 @@ class LLMMomentoCache implements INode { } } -module.exports = { nodeClass: LLMMomentoCache } +module.exports = { nodeClass: MomentoCache } diff --git a/packages/components/nodes/llmcache/LLMMomentoCache/momento.png b/packages/components/nodes/cache/MomentoCache/momento.png similarity index 100% rename from packages/components/nodes/llmcache/LLMMomentoCache/momento.png rename to packages/components/nodes/cache/MomentoCache/momento.png diff --git a/packages/components/nodes/llmcache/LLMRedisCache/LLMRedisCache.ts b/packages/components/nodes/cache/RedisCache/RedisCache.ts similarity index 85% rename from packages/components/nodes/llmcache/LLMRedisCache/LLMRedisCache.ts rename to packages/components/nodes/cache/RedisCache/RedisCache.ts index a7208977..d0143d76 100644 --- a/packages/components/nodes/llmcache/LLMRedisCache/LLMRedisCache.ts +++ b/packages/components/nodes/cache/RedisCache/RedisCache.ts @@ -1,8 +1,8 @@ import { getBaseClasses, getCredentialData, getCredentialParam, ICommonObject, INode, INodeData, INodeParams } from '../../../src' -import { RedisCache } from 'langchain/cache/ioredis' +import { RedisCache as LangchainRedisCache } from 'langchain/cache/ioredis' import { Redis } from 'ioredis' -class LLMRedisCache implements INode { +class RedisCache implements INode { label: string name: string version: number @@ -18,9 +18,9 @@ class LLMRedisCache implements INode { this.label = 'Redis LLM Cache' this.name = 'redisCache' this.version = 1.0 - this.type = 'LLMCache' + this.type = 'Cache' this.icon = 'redis.svg' - this.category = 'LLM Cache' + this.category = 'Cache' this.baseClasses = [this.type, ...getBaseClasses(RedisCache)] this.credential = { label: 'Connect Credential', @@ -45,8 +45,8 @@ class LLMRedisCache implements INode { username, password }) - return new RedisCache(client) + return new LangchainRedisCache(client) } } -module.exports = { nodeClass: LLMRedisCache } +module.exports = { nodeClass: RedisCache } diff --git a/packages/components/nodes/llmcache/LLMRedisCache/redis.svg b/packages/components/nodes/cache/RedisCache/redis.svg similarity index 100% rename from packages/components/nodes/llmcache/LLMRedisCache/redis.svg rename to packages/components/nodes/cache/RedisCache/redis.svg diff --git a/packages/components/nodes/llmcache/LLMUpstashRedisCache/LLMUpstashRedisCache.ts b/packages/components/nodes/cache/UpstashRedisCache/UpstashRedisCache.ts similarity index 82% rename from packages/components/nodes/llmcache/LLMUpstashRedisCache/LLMUpstashRedisCache.ts rename to packages/components/nodes/cache/UpstashRedisCache/UpstashRedisCache.ts index 8914b2a1..a9322b1e 100644 --- a/packages/components/nodes/llmcache/LLMUpstashRedisCache/LLMUpstashRedisCache.ts +++ b/packages/components/nodes/cache/UpstashRedisCache/UpstashRedisCache.ts @@ -1,7 +1,7 @@ import { getBaseClasses, getCredentialData, getCredentialParam, ICommonObject, INode, INodeData, INodeParams } from '../../../src' -import { UpstashRedisCache } from 'langchain/cache/upstash_redis' +import { UpstashRedisCache as LangchainUpstashRedisCache } from 'langchain/cache/upstash_redis' -class LLMUpstashRedisCache implements INode { +class UpstashRedisCache implements INode { label: string name: string version: number @@ -17,9 +17,9 @@ class LLMUpstashRedisCache implements INode { this.label = 'Upstash Redis LLM Cache' this.name = 'upstashRedisCache' this.version = 1.0 - this.type = 'LLMCache' + this.type = 'Cache' this.icon = 'upstash.png' - this.category = 'LLM Cache' + this.category = 'Cache' this.baseClasses = [this.type, ...getBaseClasses(UpstashRedisCache)] this.credential = { label: 'Connect Credential', @@ -36,7 +36,7 @@ class LLMUpstashRedisCache implements INode { const upstashConnectionUrl = getCredentialParam('upstashConnectionUrl', credentialData, nodeData) const upstashToken = getCredentialParam('upstashConnectionToken', credentialData, nodeData) - const cache = new UpstashRedisCache({ + const cache = new LangchainUpstashRedisCache({ config: { url: upstashConnectionUrl, token: upstashToken @@ -46,4 +46,4 @@ class LLMUpstashRedisCache implements INode { } } -module.exports = { nodeClass: LLMUpstashRedisCache } +module.exports = { nodeClass: UpstashRedisCache } diff --git a/packages/components/nodes/llmcache/LLMUpstashRedisCache/upstash.png b/packages/components/nodes/cache/UpstashRedisCache/upstash.png similarity index 100% rename from packages/components/nodes/llmcache/LLMUpstashRedisCache/upstash.png rename to packages/components/nodes/cache/UpstashRedisCache/upstash.png From a7574913422c518da259a35d39a85bb623871ae5 Mon Sep 17 00:00:00 2001 From: Henry Heng Date: Sun, 8 Oct 2023 11:58:37 +0100 Subject: [PATCH 15/25] Update MomentoCache.ts Base Class --- packages/components/nodes/cache/MomentoCache/MomentoCache.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/components/nodes/cache/MomentoCache/MomentoCache.ts b/packages/components/nodes/cache/MomentoCache/MomentoCache.ts index 37d90941..722f7340 100644 --- a/packages/components/nodes/cache/MomentoCache/MomentoCache.ts +++ b/packages/components/nodes/cache/MomentoCache/MomentoCache.ts @@ -21,7 +21,7 @@ class MomentoCache implements INode { this.type = 'Cache' this.icon = 'momento.png' this.category = 'Cache' - this.baseClasses = [this.type, ...getBaseClasses(MomentoCache)] + this.baseClasses = [this.type, ...getBaseClasses(LangchainMomentoCache)] this.credential = { label: 'Connect Credential', name: 'credential', From 132913f50b26524e43c4d1eb6c48619a34920c5a Mon Sep 17 00:00:00 2001 From: Henry Heng Date: Sun, 8 Oct 2023 11:58:57 +0100 Subject: [PATCH 16/25] Update RedisCache.ts Base Class --- packages/components/nodes/cache/RedisCache/RedisCache.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/components/nodes/cache/RedisCache/RedisCache.ts b/packages/components/nodes/cache/RedisCache/RedisCache.ts index d0143d76..46980acb 100644 --- a/packages/components/nodes/cache/RedisCache/RedisCache.ts +++ b/packages/components/nodes/cache/RedisCache/RedisCache.ts @@ -21,7 +21,7 @@ class RedisCache implements INode { this.type = 'Cache' this.icon = 'redis.svg' this.category = 'Cache' - this.baseClasses = [this.type, ...getBaseClasses(RedisCache)] + this.baseClasses = [this.type, ...getBaseClasses(LangchainRedisCache)] this.credential = { label: 'Connect Credential', name: 'credential', From 40b06cd43af8baf6aaf9f0b57393810245debb01 Mon Sep 17 00:00:00 2001 From: Henry Heng Date: Sun, 8 Oct 2023 11:59:14 +0100 Subject: [PATCH 17/25] Update UpstashRedisCache.ts Base Class --- .../nodes/cache/UpstashRedisCache/UpstashRedisCache.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/components/nodes/cache/UpstashRedisCache/UpstashRedisCache.ts b/packages/components/nodes/cache/UpstashRedisCache/UpstashRedisCache.ts index a9322b1e..b4ba71a5 100644 --- a/packages/components/nodes/cache/UpstashRedisCache/UpstashRedisCache.ts +++ b/packages/components/nodes/cache/UpstashRedisCache/UpstashRedisCache.ts @@ -20,7 +20,7 @@ class UpstashRedisCache implements INode { this.type = 'Cache' this.icon = 'upstash.png' this.category = 'Cache' - this.baseClasses = [this.type, ...getBaseClasses(UpstashRedisCache)] + this.baseClasses = [this.type, ...getBaseClasses(LangchainUpstashRedisCache)] this.credential = { label: 'Connect Credential', name: 'credential', From 77899b956ec16f7b1b5a1771384f182673013bb9 Mon Sep 17 00:00:00 2001 From: Henry Heng Date: Sun, 8 Oct 2023 12:00:44 +0100 Subject: [PATCH 18/25] Update index.ts --- packages/server/src/utils/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/utils/index.ts b/packages/server/src/utils/index.ts index 98802cde..664f110b 100644 --- a/packages/server/src/utils/index.ts +++ b/packages/server/src/utils/index.ts @@ -477,7 +477,7 @@ export const replaceInputsWithConfig = (flowNodeData: INodeData, overrideConfig: */ export const isStartNodeDependOnInput = (startingNodes: IReactFlowNode[], nodes: IReactFlowNode[]): boolean => { for (const node of startingNodes) { - if (node.data.category === 'LLM Cache') return true + if (node.data.category === 'Cache') return true for (const inputName in node.data.inputs) { const inputVariables = getInputVariables(node.data.inputs[inputName]) if (inputVariables.length > 0) return true From 643adb59792bb37b4716b3ce5e8922b72ebac8d9 Mon Sep 17 00:00:00 2001 From: Henry Heng Date: Sun, 8 Oct 2023 12:08:23 +0100 Subject: [PATCH 19/25] Update MomentoCache.ts type --- packages/components/nodes/cache/MomentoCache/MomentoCache.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/components/nodes/cache/MomentoCache/MomentoCache.ts b/packages/components/nodes/cache/MomentoCache/MomentoCache.ts index 722f7340..70ac9da7 100644 --- a/packages/components/nodes/cache/MomentoCache/MomentoCache.ts +++ b/packages/components/nodes/cache/MomentoCache/MomentoCache.ts @@ -18,7 +18,7 @@ class MomentoCache implements INode { this.label = 'Momento Cache' this.name = 'momentoCache' this.version = 1.0 - this.type = 'Cache' + this.type = 'MomentoCache' this.icon = 'momento.png' this.category = 'Cache' this.baseClasses = [this.type, ...getBaseClasses(LangchainMomentoCache)] From abce51353b302ef6c3138741aaa6eb73bf9388a0 Mon Sep 17 00:00:00 2001 From: Henry Heng Date: Sun, 8 Oct 2023 12:08:51 +0100 Subject: [PATCH 20/25] Update RedisCache.ts type --- packages/components/nodes/cache/RedisCache/RedisCache.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/components/nodes/cache/RedisCache/RedisCache.ts b/packages/components/nodes/cache/RedisCache/RedisCache.ts index 46980acb..c1b08be6 100644 --- a/packages/components/nodes/cache/RedisCache/RedisCache.ts +++ b/packages/components/nodes/cache/RedisCache/RedisCache.ts @@ -15,10 +15,10 @@ class RedisCache implements INode { credential: INodeParams constructor() { - this.label = 'Redis LLM Cache' + this.label = 'Redis Cache' this.name = 'redisCache' this.version = 1.0 - this.type = 'Cache' + this.type = 'RedisCache' this.icon = 'redis.svg' this.category = 'Cache' this.baseClasses = [this.type, ...getBaseClasses(LangchainRedisCache)] From 12159f6730711ab0b0690c72787305e30d823c85 Mon Sep 17 00:00:00 2001 From: Henry Heng Date: Sun, 8 Oct 2023 12:09:16 +0100 Subject: [PATCH 21/25] Update UpstashRedisCache.ts type --- .../nodes/cache/UpstashRedisCache/UpstashRedisCache.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/components/nodes/cache/UpstashRedisCache/UpstashRedisCache.ts b/packages/components/nodes/cache/UpstashRedisCache/UpstashRedisCache.ts index b4ba71a5..eb5a9e2f 100644 --- a/packages/components/nodes/cache/UpstashRedisCache/UpstashRedisCache.ts +++ b/packages/components/nodes/cache/UpstashRedisCache/UpstashRedisCache.ts @@ -14,10 +14,10 @@ class UpstashRedisCache implements INode { credential: INodeParams constructor() { - this.label = 'Upstash Redis LLM Cache' + this.label = 'Upstash Redis Cache' this.name = 'upstashRedisCache' this.version = 1.0 - this.type = 'Cache' + this.type = 'UpstashRedisCache' this.icon = 'upstash.png' this.category = 'Cache' this.baseClasses = [this.type, ...getBaseClasses(LangchainUpstashRedisCache)] From 7d4337724d6ab323a09fb96451989b115b2638b3 Mon Sep 17 00:00:00 2001 From: vinodkiran Date: Sun, 8 Oct 2023 22:59:43 +0530 Subject: [PATCH 22/25] Updating of Type LLMCache to BaseCache and renaming vars for clarity --- .../nodes/chatmodels/AWSBedrock/AWSChatBedrock.ts | 4 ++-- .../chatmodels/AzureChatOpenAI/AzureChatOpenAI.ts | 4 ++-- .../nodes/chatmodels/Bittensor/Bittensor.ts | 4 ++-- .../nodes/chatmodels/ChatAnthropic/ChatAnthropic.ts | 4 ++-- .../chatmodels/ChatGooglePaLM/ChatGooglePaLM.ts | 4 ++-- .../ChatGoogleVertexAI/ChatGoogleVertexAI.ts | 4 ++-- .../chatmodels/ChatHuggingFace/ChatHuggingFace.ts | 4 ++-- .../nodes/chatmodels/ChatLocalAI/ChatLocalAI.ts | 4 ++-- .../nodes/chatmodels/ChatOpenAI/ChatOpenAI.ts | 4 ++-- .../chatmodels/ChatOpenAICustom/ChatOpenAICustom.ts | 4 ++-- .../components/nodes/llms/AWSBedrock/AWSBedrock.ts | 13 +++++++++++-- .../nodes/llms/Azure OpenAI/AzureOpenAI.ts | 8 ++++---- .../components/nodes/llms/Bittensor/Bittensor.ts | 8 ++++---- packages/components/nodes/llms/Cohere/Cohere.ts | 8 ++++---- .../components/nodes/llms/GooglePaLM/GooglePaLM.ts | 8 ++++---- .../nodes/llms/GoogleVertexAI/GoogleVertexAI.ts | 8 ++++---- .../HuggingFaceInference/HuggingFaceInference.ts | 8 ++++---- packages/components/nodes/llms/OpenAI/OpenAI.ts | 8 ++++---- .../components/nodes/llms/Replicate/Replicate.ts | 8 ++++---- 19 files changed, 63 insertions(+), 54 deletions(-) diff --git a/packages/components/nodes/chatmodels/AWSBedrock/AWSChatBedrock.ts b/packages/components/nodes/chatmodels/AWSBedrock/AWSChatBedrock.ts index ba4bdd8a..16fbc8dc 100644 --- a/packages/components/nodes/chatmodels/AWSBedrock/AWSChatBedrock.ts +++ b/packages/components/nodes/chatmodels/AWSBedrock/AWSChatBedrock.ts @@ -44,7 +44,7 @@ class AWSChatBedrock_ChatModels implements INode { { label: 'Cache', name: 'cache', - type: 'LLMCache', + type: 'BaseCache', optional: true }, { @@ -138,7 +138,7 @@ class AWSChatBedrock_ChatModels implements INode { const iModel = nodeData.inputs?.model as string const iTemperature = nodeData.inputs?.temperature as string const iMax_tokens_to_sample = nodeData.inputs?.max_tokens_to_sample as string - const cache = nodeData.inputs?.llmCache as BaseCache + const cache = nodeData.inputs?.cache as BaseCache const obj: BaseBedrockInput & BaseLLMParams = { region: iRegion, diff --git a/packages/components/nodes/chatmodels/AzureChatOpenAI/AzureChatOpenAI.ts b/packages/components/nodes/chatmodels/AzureChatOpenAI/AzureChatOpenAI.ts index 5acf4dce..99e151e6 100644 --- a/packages/components/nodes/chatmodels/AzureChatOpenAI/AzureChatOpenAI.ts +++ b/packages/components/nodes/chatmodels/AzureChatOpenAI/AzureChatOpenAI.ts @@ -36,7 +36,7 @@ class AzureChatOpenAI_ChatModels implements INode { { label: 'Cache', name: 'cache', - type: 'LLMCache', + type: 'BaseCache', optional: true }, { @@ -115,7 +115,7 @@ class AzureChatOpenAI_ChatModels implements INode { const presencePenalty = nodeData.inputs?.presencePenalty as string const timeout = nodeData.inputs?.timeout as string const streaming = nodeData.inputs?.streaming as boolean - const cache = nodeData.inputs?.llmCache as BaseCache + const cache = nodeData.inputs?.cache as BaseCache const credentialData = await getCredentialData(nodeData.credential ?? '', options) const azureOpenAIApiKey = getCredentialParam('azureOpenAIApiKey', credentialData, nodeData) diff --git a/packages/components/nodes/chatmodels/Bittensor/Bittensor.ts b/packages/components/nodes/chatmodels/Bittensor/Bittensor.ts index 89c37157..36b084e6 100644 --- a/packages/components/nodes/chatmodels/Bittensor/Bittensor.ts +++ b/packages/components/nodes/chatmodels/Bittensor/Bittensor.ts @@ -27,7 +27,7 @@ class Bittensor_ChatModels implements INode { { label: 'Cache', name: 'cache', - type: 'LLMCache', + type: 'BaseCache', optional: true }, { @@ -42,7 +42,7 @@ class Bittensor_ChatModels implements INode { async init(nodeData: INodeData, _: string): Promise { const system_prompt = nodeData.inputs?.system_prompt as string - const cache = nodeData.inputs?.llmCache as BaseCache + const cache = nodeData.inputs?.cache as BaseCache const obj: Partial = { systemPrompt: system_prompt diff --git a/packages/components/nodes/chatmodels/ChatAnthropic/ChatAnthropic.ts b/packages/components/nodes/chatmodels/ChatAnthropic/ChatAnthropic.ts index 039b437b..f16968b6 100644 --- a/packages/components/nodes/chatmodels/ChatAnthropic/ChatAnthropic.ts +++ b/packages/components/nodes/chatmodels/ChatAnthropic/ChatAnthropic.ts @@ -35,7 +35,7 @@ class ChatAnthropic_ChatModels implements INode { { label: 'Cache', name: 'cache', - type: 'LLMCache', + type: 'BaseCache', optional: true }, { @@ -143,7 +143,7 @@ class ChatAnthropic_ChatModels implements INode { const topP = nodeData.inputs?.topP as string const topK = nodeData.inputs?.topK as string const streaming = nodeData.inputs?.streaming as boolean - const cache = nodeData.inputs?.llmCache as BaseCache + const cache = nodeData.inputs?.cache as BaseCache const credentialData = await getCredentialData(nodeData.credential ?? '', options) const anthropicApiKey = getCredentialParam('anthropicApiKey', credentialData, nodeData) diff --git a/packages/components/nodes/chatmodels/ChatGooglePaLM/ChatGooglePaLM.ts b/packages/components/nodes/chatmodels/ChatGooglePaLM/ChatGooglePaLM.ts index 8ee3fbb8..121406c7 100644 --- a/packages/components/nodes/chatmodels/ChatGooglePaLM/ChatGooglePaLM.ts +++ b/packages/components/nodes/chatmodels/ChatGooglePaLM/ChatGooglePaLM.ts @@ -34,7 +34,7 @@ class ChatGooglePaLM_ChatModels implements INode { { label: 'Cache', name: 'cache', - type: 'LLMCache', + type: 'BaseCache', optional: true }, { @@ -103,7 +103,7 @@ class ChatGooglePaLM_ChatModels implements INode { const temperature = nodeData.inputs?.temperature as string const topP = nodeData.inputs?.topP as string const topK = nodeData.inputs?.topK as string - const cache = nodeData.inputs?.llmCache as BaseCache + const cache = nodeData.inputs?.cache as BaseCache const credentialData = await getCredentialData(nodeData.credential ?? '', options) const googleMakerSuiteKey = getCredentialParam('googleMakerSuiteKey', credentialData, nodeData) diff --git a/packages/components/nodes/chatmodels/ChatGoogleVertexAI/ChatGoogleVertexAI.ts b/packages/components/nodes/chatmodels/ChatGoogleVertexAI/ChatGoogleVertexAI.ts index 4f876239..6b070bd9 100644 --- a/packages/components/nodes/chatmodels/ChatGoogleVertexAI/ChatGoogleVertexAI.ts +++ b/packages/components/nodes/chatmodels/ChatGoogleVertexAI/ChatGoogleVertexAI.ts @@ -38,7 +38,7 @@ class GoogleVertexAI_ChatModels implements INode { { label: 'Cache', name: 'cache', - type: 'LLMCache', + type: 'BaseCache', optional: true }, { @@ -120,7 +120,7 @@ class GoogleVertexAI_ChatModels implements INode { const modelName = nodeData.inputs?.modelName as string const maxOutputTokens = nodeData.inputs?.maxOutputTokens as string const topP = nodeData.inputs?.topP as string - const cache = nodeData.inputs?.llmCache as BaseCache + const cache = nodeData.inputs?.cache as BaseCache const obj: GoogleVertexAIChatInput = { temperature: parseFloat(temperature), diff --git a/packages/components/nodes/chatmodels/ChatHuggingFace/ChatHuggingFace.ts b/packages/components/nodes/chatmodels/ChatHuggingFace/ChatHuggingFace.ts index d9c0d22f..153c5d10 100644 --- a/packages/components/nodes/chatmodels/ChatHuggingFace/ChatHuggingFace.ts +++ b/packages/components/nodes/chatmodels/ChatHuggingFace/ChatHuggingFace.ts @@ -34,7 +34,7 @@ class ChatHuggingFace_ChatModels implements INode { { label: 'Cache', name: 'cache', - type: 'LLMCache', + type: 'BaseCache', optional: true }, { @@ -109,7 +109,7 @@ class ChatHuggingFace_ChatModels implements INode { const hfTopK = nodeData.inputs?.hfTopK as string const frequencyPenalty = nodeData.inputs?.frequencyPenalty as string const endpoint = nodeData.inputs?.endpoint as string - const cache = nodeData.inputs?.llmCache as BaseCache + const cache = nodeData.inputs?.cache as BaseCache const credentialData = await getCredentialData(nodeData.credential ?? '', options) const huggingFaceApiKey = getCredentialParam('huggingFaceApiKey', credentialData, nodeData) diff --git a/packages/components/nodes/chatmodels/ChatLocalAI/ChatLocalAI.ts b/packages/components/nodes/chatmodels/ChatLocalAI/ChatLocalAI.ts index 78fe31dd..18ed409b 100644 --- a/packages/components/nodes/chatmodels/ChatLocalAI/ChatLocalAI.ts +++ b/packages/components/nodes/chatmodels/ChatLocalAI/ChatLocalAI.ts @@ -29,7 +29,7 @@ class ChatLocalAI_ChatModels implements INode { { label: 'Cache', name: 'cache', - type: 'LLMCache', + type: 'BaseCache', optional: true }, { @@ -86,7 +86,7 @@ class ChatLocalAI_ChatModels implements INode { const topP = nodeData.inputs?.topP as string const timeout = nodeData.inputs?.timeout as string const basePath = nodeData.inputs?.basePath as string - const cache = nodeData.inputs?.llmCache as BaseCache + const cache = nodeData.inputs?.cache as BaseCache const obj: Partial & BaseLLMParams & { openAIApiKey?: string } = { temperature: parseFloat(temperature), diff --git a/packages/components/nodes/chatmodels/ChatOpenAI/ChatOpenAI.ts b/packages/components/nodes/chatmodels/ChatOpenAI/ChatOpenAI.ts index a9eae481..f74bd642 100644 --- a/packages/components/nodes/chatmodels/ChatOpenAI/ChatOpenAI.ts +++ b/packages/components/nodes/chatmodels/ChatOpenAI/ChatOpenAI.ts @@ -35,7 +35,7 @@ class ChatOpenAI_ChatModels implements INode { { label: 'Cache', name: 'cache', - type: 'LLMCache', + type: 'BaseCache', optional: true }, { @@ -159,7 +159,7 @@ class ChatOpenAI_ChatModels implements INode { const credentialData = await getCredentialData(nodeData.credential ?? '', options) const openAIApiKey = getCredentialParam('openAIApiKey', credentialData, nodeData) - const cache = nodeData.inputs?.llmCache as BaseCache + const cache = nodeData.inputs?.cache as BaseCache const obj: Partial & BaseLLMParams & { openAIApiKey?: string } = { temperature: parseFloat(temperature), diff --git a/packages/components/nodes/chatmodels/ChatOpenAICustom/ChatOpenAICustom.ts b/packages/components/nodes/chatmodels/ChatOpenAICustom/ChatOpenAICustom.ts index 6f8d1310..2a01a2e5 100644 --- a/packages/components/nodes/chatmodels/ChatOpenAICustom/ChatOpenAICustom.ts +++ b/packages/components/nodes/chatmodels/ChatOpenAICustom/ChatOpenAICustom.ts @@ -36,7 +36,7 @@ class ChatOpenAICustom_ChatModels implements INode { { label: 'Cache', name: 'cache', - type: 'LLMCache', + type: 'BaseCache', optional: true }, { @@ -121,7 +121,7 @@ class ChatOpenAICustom_ChatModels implements INode { const streaming = nodeData.inputs?.streaming as boolean const basePath = nodeData.inputs?.basepath as string const baseOptions = nodeData.inputs?.baseOptions - const cache = nodeData.inputs?.llmCache as BaseCache + const cache = nodeData.inputs?.cache as BaseCache const credentialData = await getCredentialData(nodeData.credential ?? '', options) const openAIApiKey = getCredentialParam('openAIApiKey', credentialData, nodeData) diff --git a/packages/components/nodes/llms/AWSBedrock/AWSBedrock.ts b/packages/components/nodes/llms/AWSBedrock/AWSBedrock.ts index 68ee7ed2..8f57daac 100644 --- a/packages/components/nodes/llms/AWSBedrock/AWSBedrock.ts +++ b/packages/components/nodes/llms/AWSBedrock/AWSBedrock.ts @@ -2,6 +2,8 @@ import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Inter import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils' import { Bedrock } from 'langchain/llms/bedrock' import { BaseBedrockInput } from 'langchain/dist/util/bedrock' +import { BaseCache } from 'langchain/schema' +import { BaseLLMParams } from 'langchain/llms/base' /** * I had to run the following to build the component @@ -39,6 +41,12 @@ class AWSBedrock_LLMs implements INode { optional: true } this.inputs = [ + { + label: 'Cache', + name: 'cache', + type: 'BaseCache', + optional: true + }, { label: 'Region', name: 'region', @@ -130,8 +138,8 @@ class AWSBedrock_LLMs implements INode { const iModel = nodeData.inputs?.model as string const iTemperature = nodeData.inputs?.temperature as string const iMax_tokens_to_sample = nodeData.inputs?.max_tokens_to_sample as string - - const obj: Partial = { + const cache = nodeData.inputs?.cache as BaseCache + const obj: Partial & BaseLLMParams = { model: iModel, region: iRegion, temperature: parseFloat(iTemperature), @@ -157,6 +165,7 @@ class AWSBedrock_LLMs implements INode { sessionToken: credentialApiSession } } + if (cache) obj.cache = cache const amazonBedrock = new Bedrock(obj) return amazonBedrock diff --git a/packages/components/nodes/llms/Azure OpenAI/AzureOpenAI.ts b/packages/components/nodes/llms/Azure OpenAI/AzureOpenAI.ts index e5cca769..f50e3d95 100644 --- a/packages/components/nodes/llms/Azure OpenAI/AzureOpenAI.ts +++ b/packages/components/nodes/llms/Azure OpenAI/AzureOpenAI.ts @@ -33,8 +33,8 @@ class AzureOpenAI_LLMs implements INode { this.inputs = [ { label: 'Cache', - name: 'llmCache', - type: 'LLMCache', + name: 'cache', + type: 'BaseCache', optional: true }, { @@ -170,7 +170,7 @@ class AzureOpenAI_LLMs implements INode { const azureOpenAIApiDeploymentName = getCredentialParam('azureOpenAIApiDeploymentName', credentialData, nodeData) const azureOpenAIApiVersion = getCredentialParam('azureOpenAIApiVersion', credentialData, nodeData) - const llmCache = nodeData.inputs?.llmCache as BaseCache + const cache = nodeData.inputs?.cache as BaseCache const obj: Partial & BaseLLMParams & Partial = { temperature: parseFloat(temperature), @@ -188,7 +188,7 @@ class AzureOpenAI_LLMs implements INode { if (presencePenalty) obj.presencePenalty = parseFloat(presencePenalty) if (timeout) obj.timeout = parseInt(timeout, 10) if (bestOf) obj.bestOf = parseInt(bestOf, 10) - if (llmCache) obj.cache = llmCache + if (cache) obj.cache = cache const model = new OpenAI(obj) return model diff --git a/packages/components/nodes/llms/Bittensor/Bittensor.ts b/packages/components/nodes/llms/Bittensor/Bittensor.ts index 84401e59..e6cc2bb6 100644 --- a/packages/components/nodes/llms/Bittensor/Bittensor.ts +++ b/packages/components/nodes/llms/Bittensor/Bittensor.ts @@ -27,8 +27,8 @@ class Bittensor_LLMs implements INode { this.inputs = [ { label: 'Cache', - name: 'llmCache', - type: 'LLMCache', + name: 'cache', + type: 'BaseCache', optional: true }, { @@ -52,13 +52,13 @@ class Bittensor_LLMs implements INode { async init(nodeData: INodeData, _: string): Promise { const system_prompt = nodeData.inputs?.system_prompt as string const topResponses = Number(nodeData.inputs?.topResponses as number) - const llmCache = nodeData.inputs?.llmCache as BaseCache + const cache = nodeData.inputs?.cache as BaseCache const obj: Partial & BaseLLMParams = { systemPrompt: system_prompt, topResponses: topResponses } - if (llmCache) obj.cache = llmCache + if (cache) obj.cache = cache const model = new NIBittensorLLM(obj) return model diff --git a/packages/components/nodes/llms/Cohere/Cohere.ts b/packages/components/nodes/llms/Cohere/Cohere.ts index 8b4c0ac2..3fde0af0 100644 --- a/packages/components/nodes/llms/Cohere/Cohere.ts +++ b/packages/components/nodes/llms/Cohere/Cohere.ts @@ -33,8 +33,8 @@ class Cohere_LLMs implements INode { this.inputs = [ { label: 'Cache', - name: 'llmCache', - type: 'LLMCache', + name: 'cache', + type: 'BaseCache', optional: true }, { @@ -92,7 +92,7 @@ class Cohere_LLMs implements INode { const temperature = nodeData.inputs?.temperature as string const modelName = nodeData.inputs?.modelName as string const maxTokens = nodeData.inputs?.maxTokens as string - const llmCache = nodeData.inputs?.llmCache as BaseCache + const cache = nodeData.inputs?.cache as BaseCache const credentialData = await getCredentialData(nodeData.credential ?? '', options) const cohereApiKey = getCredentialParam('cohereApiKey', credentialData, nodeData) @@ -103,7 +103,7 @@ class Cohere_LLMs implements INode { if (maxTokens) obj.maxTokens = parseInt(maxTokens, 10) if (modelName) obj.model = modelName if (temperature) obj.temperature = parseFloat(temperature) - if (llmCache) obj.cache = llmCache + if (cache) obj.cache = cache const model = new Cohere(obj) return model } diff --git a/packages/components/nodes/llms/GooglePaLM/GooglePaLM.ts b/packages/components/nodes/llms/GooglePaLM/GooglePaLM.ts index d916c09f..d3212a1c 100644 --- a/packages/components/nodes/llms/GooglePaLM/GooglePaLM.ts +++ b/packages/components/nodes/llms/GooglePaLM/GooglePaLM.ts @@ -32,8 +32,8 @@ class GooglePaLM_LLMs implements INode { this.inputs = [ { label: 'Cache', - name: 'llmCache', - type: 'LLMCache', + name: 'cache', + type: 'BaseCache', optional: true }, { @@ -132,7 +132,7 @@ class GooglePaLM_LLMs implements INode { const topP = nodeData.inputs?.topP as string const topK = nodeData.inputs?.topK as string const stopSequencesObj = nodeData.inputs?.stopSequencesObj - const llmCache = nodeData.inputs?.llmCache as BaseCache + const cache = nodeData.inputs?.cache as BaseCache const credentialData = await getCredentialData(nodeData.credential ?? '', options) const googleMakerSuiteKey = getCredentialParam('googleMakerSuiteKey', credentialData, nodeData) @@ -146,7 +146,7 @@ class GooglePaLM_LLMs implements INode { if (maxOutputTokens) obj.maxOutputTokens = parseInt(maxOutputTokens, 10) if (topP) obj.topP = parseFloat(topP) if (topK) obj.topK = parseFloat(topK) - if (llmCache) obj.cache = llmCache + if (cache) obj.cache = cache let parsedStopSequences: any | undefined = undefined if (stopSequencesObj) { diff --git a/packages/components/nodes/llms/GoogleVertexAI/GoogleVertexAI.ts b/packages/components/nodes/llms/GoogleVertexAI/GoogleVertexAI.ts index 41b18475..6b6d534b 100644 --- a/packages/components/nodes/llms/GoogleVertexAI/GoogleVertexAI.ts +++ b/packages/components/nodes/llms/GoogleVertexAI/GoogleVertexAI.ts @@ -37,8 +37,8 @@ class GoogleVertexAI_LLMs implements INode { this.inputs = [ { label: 'Cache', - name: 'llmCache', - type: 'LLMCache', + name: 'cache', + type: 'BaseCache', optional: true }, { @@ -127,7 +127,7 @@ class GoogleVertexAI_LLMs implements INode { const modelName = nodeData.inputs?.modelName as string const maxOutputTokens = nodeData.inputs?.maxOutputTokens as string const topP = nodeData.inputs?.topP as string - const llmCache = nodeData.inputs?.llmCache as BaseCache + const cache = nodeData.inputs?.cache as BaseCache const obj: Partial = { temperature: parseFloat(temperature), @@ -137,7 +137,7 @@ class GoogleVertexAI_LLMs implements INode { if (maxOutputTokens) obj.maxOutputTokens = parseInt(maxOutputTokens, 10) if (topP) obj.topP = parseFloat(topP) - if (llmCache) obj.cache = llmCache + if (cache) obj.cache = cache const model = new GoogleVertexAI(obj) return model diff --git a/packages/components/nodes/llms/HuggingFaceInference/HuggingFaceInference.ts b/packages/components/nodes/llms/HuggingFaceInference/HuggingFaceInference.ts index 9d6226e2..8dcf021b 100644 --- a/packages/components/nodes/llms/HuggingFaceInference/HuggingFaceInference.ts +++ b/packages/components/nodes/llms/HuggingFaceInference/HuggingFaceInference.ts @@ -33,8 +33,8 @@ class HuggingFaceInference_LLMs implements INode { this.inputs = [ { label: 'Cache', - name: 'llmCache', - type: 'LLMCache', + name: 'cache', + type: 'BaseCache', optional: true }, { @@ -113,7 +113,7 @@ class HuggingFaceInference_LLMs implements INode { const credentialData = await getCredentialData(nodeData.credential ?? '', options) const huggingFaceApiKey = getCredentialParam('huggingFaceApiKey', credentialData, nodeData) - const llmCache = nodeData.inputs?.llmCache as BaseCache + const cache = nodeData.inputs?.cache as BaseCache const obj: Partial = { model, @@ -128,7 +128,7 @@ class HuggingFaceInference_LLMs implements INode { if (endpoint) obj.endpoint = endpoint const huggingFace = new HuggingFaceInference(obj) - if (llmCache) huggingFace.cache = llmCache + if (cache) huggingFace.cache = cache return huggingFace } diff --git a/packages/components/nodes/llms/OpenAI/OpenAI.ts b/packages/components/nodes/llms/OpenAI/OpenAI.ts index 607ebeb4..9109dd40 100644 --- a/packages/components/nodes/llms/OpenAI/OpenAI.ts +++ b/packages/components/nodes/llms/OpenAI/OpenAI.ts @@ -34,8 +34,8 @@ class OpenAI_LLMs implements INode { this.inputs = [ { label: 'Cache', - name: 'llmCache', - type: 'LLMCache', + name: 'cache', + type: 'BaseCache', optional: true }, { @@ -157,7 +157,7 @@ class OpenAI_LLMs implements INode { const credentialData = await getCredentialData(nodeData.credential ?? '', options) const openAIApiKey = getCredentialParam('openAIApiKey', credentialData, nodeData) - const llmCache = nodeData.inputs?.llmCache as BaseCache + const cache = nodeData.inputs?.cache as BaseCache const obj: Partial & BaseLLMParams & { openAIApiKey?: string } = { temperature: parseFloat(temperature), @@ -174,7 +174,7 @@ class OpenAI_LLMs implements INode { if (batchSize) obj.batchSize = parseInt(batchSize, 10) if (bestOf) obj.bestOf = parseInt(bestOf, 10) - if (llmCache) obj.cache = llmCache + if (cache) obj.cache = cache let parsedBaseOptions: any | undefined = undefined if (baseOptions) { diff --git a/packages/components/nodes/llms/Replicate/Replicate.ts b/packages/components/nodes/llms/Replicate/Replicate.ts index fd1eb6f7..fd5373a1 100644 --- a/packages/components/nodes/llms/Replicate/Replicate.ts +++ b/packages/components/nodes/llms/Replicate/Replicate.ts @@ -34,8 +34,8 @@ class Replicate_LLMs implements INode { this.inputs = [ { label: 'Cache', - name: 'llmCache', - type: 'LLMCache', + name: 'cache', + type: 'BaseCache', optional: true }, { @@ -111,7 +111,7 @@ class Replicate_LLMs implements INode { const name = modelName.split(':')[0].split('/').pop() const org = modelName.split(':')[0].split('/')[0] - const llmCache = nodeData.inputs?.llmCache as BaseCache + const cache = nodeData.inputs?.cache as BaseCache const obj: ReplicateInput & BaseLLMParams = { model: `${org}/${name}:${version}`, @@ -130,7 +130,7 @@ class Replicate_LLMs implements INode { } if (Object.keys(inputs).length) obj.input = inputs - if (llmCache) obj.cache = llmCache + if (cache) obj.cache = cache const model = new Replicate(obj) return model From 644bd09ef2a659687daa6431a52edca8ca693a7b Mon Sep 17 00:00:00 2001 From: vinodkiran Date: Sun, 8 Oct 2023 23:03:00 +0530 Subject: [PATCH 23/25] Removing InMemoryCache and code cleanup in MomentoCache --- .../cache/InMemoryCache/InMemoryCache.ts | 36 ------------------ .../nodes/cache/InMemoryCache/memorycache.png | Bin 2717 -> 0 bytes .../nodes/cache/MomentoCache/MomentoCache.ts | 1 - 3 files changed, 37 deletions(-) delete mode 100644 packages/components/nodes/cache/InMemoryCache/InMemoryCache.ts delete mode 100644 packages/components/nodes/cache/InMemoryCache/memorycache.png diff --git a/packages/components/nodes/cache/InMemoryCache/InMemoryCache.ts b/packages/components/nodes/cache/InMemoryCache/InMemoryCache.ts deleted file mode 100644 index 8bed721f..00000000 --- a/packages/components/nodes/cache/InMemoryCache/InMemoryCache.ts +++ /dev/null @@ -1,36 +0,0 @@ -import { INode, INodeParams } from '../../../src/Interface' -import { InMemoryCache as LangchainInMemoryCache } from 'langchain/cache' -import { getBaseClasses } from '../../../src' - -class InMemoryCache implements INode { - label: string - name: string - version: number - description: string - type: string - icon: string - category: string - baseClasses: string[] - inputs: INodeParams[] - inMemoryCache: any - - constructor() { - this.label = 'Local (Builtin) Cache' - this.name = 'localCache' - this.version = 1.0 - this.type = 'Cache' - this.icon = 'memorycache.png' - this.category = 'Cache' - this.baseClasses = [this.type, ...getBaseClasses(InMemoryCache)] - this.inputs = [] - } - - async init(): Promise { - if (!this.inMemoryCache) { - this.inMemoryCache = LangchainInMemoryCache.global() - } - return this.inMemoryCache - } -} - -module.exports = { nodeClass: InMemoryCache } diff --git a/packages/components/nodes/cache/InMemoryCache/memorycache.png b/packages/components/nodes/cache/InMemoryCache/memorycache.png deleted file mode 100644 index aaeecd6fb100be3a10b3d1faca046f9fa409f018..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2717 zcmV;O3S#w%P)Px#1ZP1_K>z@;j|==^1poj532;bRa{vG=N&o;XN&$CzbWH#N3NlGVK~!i%-C283 zROc1{vCA&(gIyj10)qh}C=crxZQ5yO635s{6UQ-WYc%!I^ba*Q6JNDcCz}4D@m1TI zDEO$24-_rdM;v3EJghMqry4;Ul?7k;Ho7411s0a2=bU@*x*~*YY%KOSbGUoI``zz5 z_q@+XNIL;4s%vTxYBZ{MJrB%kXlN*`R;%0#^WWUOiTMi_;ABw|!otI8mOSfmVCD^> zM$CKX9Sk2nTw#9c(xsR)XATl$tbE&FA1gRoC+yb-NOwtd7(PR=8ndoY!dz0A|AS0PpIBCcn zri@!?F;-76UK}kgK0Bo z!qeEOjy^b8>rSTwKOH&@&CSj7K}MrNVV<~_32pv1?B2BtrLGd3JA01q{n)fQAJI|K zICku)GL=wUU5&gIEBG~(Uo3}-?2r*`9=slJQgplsx$iAUTU(pVe^Br%Y%Xo>ZNhl4 z`HIcYeS02Ou6Pgeaq(y*X|;G;#8fP#s)mLJ`3N=v4Qx)H`g*j}LTny<-zj6XPfJUS zXy9RCpuKE%yD)$2)-8N}=Cnwwea7?|y!B&6B_*zQk$h=cDO|2na#oY_s3(IY-{|pR z#E21!*;G-qsjaOg+f`klhl9gqT52i@nlSCHw?q}>cI{-Q4@4AeJ01VNi1}o4m6Y(_ zk1rl;<^!Q}Ro)6TH#JMF=9o+>>K1;|U{qw3WIcZGQGoZkefu^}m7Gx|&#|+H;`a8P zJJ9TFMQghsEjq|Y3ypf^Wq2qTJW6PSQY1fj9{#jwAzt{|ix@a)5Sl2Jv*eXYA@Xrl zUdam#9_KO6dHz%pnR0XJJGZ(A#a#2mv*8`_b$#jf}hzKd>9uljYbzQ_TlKU?U0t2z z429}YM^~oWI=u~bK2~<{AqPC_MUawNR@>;x_(O?vZci0_>h;pJ& zaxz}|#m_Nn)XSu)-V&`|{pDz7al-Lq$KY@})w_GqHS@8=d{9&o(B#_sIxPFkpVa~2 z)iGl+W%6X~-mwb}-X=Jxld_ZJvUUvABv2e8=9Rg8k=By0q(!!ME|aUPs?kcBB0cR% zEL^e}qetImKBr$M>Kp3CpE@F#`ED~TZ)>7XD=T|IfUaD=gn}(wgz;EvRH9>Iq%D*V zhoo#UDB)Ei5)s=o6Mr1rxShuFS&AuvuvM{jA?niPw&C<0fEhVWBj~Jm=#13)Dioq(N@? zw~NDagU-_C^Pr6W{gIxYE)L1Ke(f5LQhS-dF(2#Juahx`Dbyt9)*W;)&kM0BXyB-{ zV96qE+K`W!=okzbI1nkRDa2F^o}eo`VWcw>5e^4@lmj?KdKx`A{q-q)`pGA_b@Lku z!&k3f!S(Amkd%}t#~BZbZXK4@*XU`)$NN81M}XawsB$T^hFQXqkd%b75*MuD7SaYs z7(UNwho4l>j>^5KX7>!f=d&JmY%McKMMvW3SI6ZyV?1f}%P$Vzqiqe$k8X$U46K8C z3U8C9yn&+PVr-;1%^5T@(kT@tOP!sV%_EAgoP>CcpK%C`a70KB363T#GL7^?Kl7>t znQCflSOhqKwhV4}1&$m(3@b4c0MAnq_;q*aJ~Y&O&_d>=kzVL#o-0j_CjWM-oL+6) zwhd{?eXw!;1~}NhDVecanL2{UIL`B&77BwNN_rfDuHC$WpFHynob`E1e$9mwtcRJ1`ip6 zDO0E7`i+~|w534ZMT0)$(G_hNY@-?$zq<&RE|p{AZ(hgW*1s=fEX6CAFT-8orjE=@ zq#QJH0N}Gtn_8$PB_<)Q?~`O%ArgfOHgCqELx-_$!+M#|FdFr-rJH$<4jMT8ZKVvm zch4R)QmW6%%tU5(4zjbesJcu>X8%mYQ=ukiPJevfR-B>uUsiS&A5mg+opGV0)I}w8 z4g6#xadELQ2NiEYF7vI>@32APfTM(+B$<$yh}S1h5L4N+`vbf?e?ERaegd8vGMFTs zh(xA@MD$5{0x!MvBBs6hCf2U`D~gMb6VpCKI-KGw;TDT%B2ZH3gf8Ygv%sdtCXAmj z5v$j(Mc$g#l1&q+mQP7ZL3(N$GN|ARroJhuNTzBshC(MtCa!b3X7s-Duv6%ld=JF` zKl5Cnvzj$(@YKt(_rZe)q)J>v%}FF>PLAWj@jRHiivL`ZvdQ`Q9hHpgd4G4%#XMCh zxNzY@fZia-OYheYpa1g!)#bhM*~g#4VzUZ!n$u}DyhhqM{{w^1>p8S?jlv%`a^y&< z@in@7`D&+)NJn%VDtSYW(|?>dR~-Qg3jYpQS*e^PaQjzWTm)B1sZ@zRbxv1^u-K5% zF9SIPaxm}*1Mvbyp{%Sd%$xhRIs;U3mLZCo`R>qt8>oVtW(^J0OguMqD3&f;MvY0P zjIlJi7vhHY=;|b{$E#FSs4yf zG3WKQ!eTNbp1La5t{yx*^D0C-v>0oJCC?>+r={!O(?lP^O$R3>{$Cc`wdUk{aUb(@ z{xAot@^ZU8`p^URjo@on(ANL1`|x|$sC_IU?FsYRf4~!Z#XXSZA2nDotYJPM{pv)q z!aR5I3l}ZIN$QHs<}fjZo(Mlh0IM||vq{j;4;!X1uRfE*QN51yzW|>|goT?G0R#9C XQt;1#;KI))00000NkvXXu0mjfl~Wzs diff --git a/packages/components/nodes/cache/MomentoCache/MomentoCache.ts b/packages/components/nodes/cache/MomentoCache/MomentoCache.ts index 70ac9da7..9aa82e82 100644 --- a/packages/components/nodes/cache/MomentoCache/MomentoCache.ts +++ b/packages/components/nodes/cache/MomentoCache/MomentoCache.ts @@ -36,7 +36,6 @@ class MomentoCache implements INode { const credentialData = await getCredentialData(nodeData.credential ?? '', options) const apiKey = getCredentialParam('momentoApiKey', credentialData, nodeData) const cacheName = getCredentialParam('momentoCache', credentialData, nodeData) - const endPoint = getCredentialParam('momentoEndpoint', credentialData, nodeData) // See https://github.com/momentohq/client-sdk-javascript for connection options const client = new CacheClient({ From 303e5653dd403f408dbac04011b49af7ff5fb972 Mon Sep 17 00:00:00 2001 From: Henry Heng Date: Mon, 9 Oct 2023 16:44:04 +0100 Subject: [PATCH 24/25] update cacheValue to capture other outputs object --- packages/components/src/handler.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/components/src/handler.ts b/packages/components/src/handler.ts index a102b473..1ae16050 100644 --- a/packages/components/src/handler.ts +++ b/packages/components/src/handler.ts @@ -191,7 +191,7 @@ export class CustomChainHandler extends BaseCallbackHandler { Callback Order is "Chain Start -> Chain End" for cached responses. */ if (this.cachedResponse) { - const cachedValue = outputs.text as string + const cachedValue: string = outputs.text ?? outputs.response ?? outputs.output ?? '' //split at whitespace, and keep the whitespace. This is to preserve the original formatting. const result = cachedValue.split(/(\s+)/) result.forEach((token: string, index: number) => { From 719c8c4e55d8f570918c2c0aa2a4c35893591ea1 Mon Sep 17 00:00:00 2001 From: Henry Heng Date: Mon, 9 Oct 2023 18:25:36 +0100 Subject: [PATCH 25/25] only emit token when parentId is undefined --- packages/components/src/handler.ts | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/packages/components/src/handler.ts b/packages/components/src/handler.ts index 1ae16050..3b1952d6 100644 --- a/packages/components/src/handler.ts +++ b/packages/components/src/handler.ts @@ -177,21 +177,17 @@ export class CustomChainHandler extends BaseCallbackHandler { } handleLLMEnd() { - /* send the end event from handleChainEnd */ - // this.socketIO.to(this.socketIOClientId).emit('end') + this.socketIO.to(this.socketIOClientId).emit('end') } - handleChainEnd(outputs: ChainValues): void | Promise { - if (this.returnSourceDocuments) { - this.socketIO.to(this.socketIOClientId).emit('sourceDocuments', outputs?.sourceDocuments) - } + handleChainEnd(outputs: ChainValues, _: string, parentRunId?: string): void | Promise { /* Langchain does not call handleLLMStart, handleLLMEnd, handleLLMNewToken when the chain is cached. Callback Order is "Chain Start -> LLM Start --> LLM Token --> LLM End -> Chain End" for normal responses. Callback Order is "Chain Start -> Chain End" for cached responses. */ - if (this.cachedResponse) { - const cachedValue: string = outputs.text ?? outputs.response ?? outputs.output ?? '' + if (this.cachedResponse && parentRunId === undefined) { + const cachedValue = outputs.text ?? outputs.response ?? outputs.output ?? outputs.output_text //split at whitespace, and keep the whitespace. This is to preserve the original formatting. const result = cachedValue.split(/(\s+)/) result.forEach((token: string, index: number) => { @@ -200,8 +196,15 @@ export class CustomChainHandler extends BaseCallbackHandler { } this.socketIO.to(this.socketIOClientId).emit('token', token) }) + if (this.returnSourceDocuments) { + this.socketIO.to(this.socketIOClientId).emit('sourceDocuments', outputs?.sourceDocuments) + } + this.socketIO.to(this.socketIOClientId).emit('end') + } else { + if (this.returnSourceDocuments) { + this.socketIO.to(this.socketIOClientId).emit('sourceDocuments', outputs?.sourceDocuments) + } } - this.socketIO.to(this.socketIOClientId).emit('end') } }