diff --git a/packages/components/credentials/MomentoCacheApi.credential.ts b/packages/components/credentials/MomentoCacheApi.credential.ts new file mode 100644 index 00000000..038f826d --- /dev/null +++ b/packages/components/credentials/MomentoCacheApi.credential.ts @@ -0,0 +1,36 @@ +import { INodeParams, INodeCredential } from '../src/Interface' + +class MomentoCacheApi implements INodeCredential { + label: string + name: string + version: number + description: string + inputs: INodeParams[] + + constructor() { + this.label = 'Momento Cache API' + this.name = 'momentoCacheApi' + this.version = 1.0 + this.description = + 'Refer to official guide on how to get API key on Momento' + this.inputs = [ + { + label: 'Cache', + name: 'momentoCache', + type: 'string' + }, + { + label: 'API Key', + name: 'momentoApiKey', + type: 'password' + }, + { + label: 'Endpoint', + name: 'momentoEndpoint', + type: 'string' + } + ] + } +} + +module.exports = { credClass: MomentoCacheApi } diff --git a/packages/components/credentials/RedisCacheApi.credential.ts b/packages/components/credentials/RedisCacheApi.credential.ts new file mode 100644 index 00000000..e09a94e7 --- /dev/null +++ b/packages/components/credentials/RedisCacheApi.credential.ts @@ -0,0 +1,43 @@ +import { INodeParams, INodeCredential } from '../src/Interface' + +class RedisCacheApi implements INodeCredential { + label: string + name: string + version: number + description: string + inputs: INodeParams[] + + constructor() { + this.label = 'Redis Cache API' + this.name = 'redisCacheApi' + this.version = 1.0 + this.inputs = [ + { + label: 'Redis Host', + name: 'redisCacheHost', + type: 'string', + default: '127.0.0.1' + }, + { + label: 'Port', + name: 'redisCachePort', + type: 'number', + default: '6789' + }, + { + label: 'User', + name: 'redisCacheUser', + type: 'string', + placeholder: '' + }, + { + label: 'Password', + name: 'redisCachePwd', + type: 'password', + placeholder: '' + } + ] + } +} + +module.exports = { credClass: RedisCacheApi } diff --git a/packages/components/credentials/UpstashRedisApi.credential.ts b/packages/components/credentials/UpstashRedisApi.credential.ts new file mode 100644 index 00000000..b6e62ff3 --- /dev/null +++ b/packages/components/credentials/UpstashRedisApi.credential.ts @@ -0,0 +1,28 @@ +import { INodeParams, INodeCredential } from '../src/Interface' + +class UpstashRedisApi implements INodeCredential { + label: string + name: string + version: number + inputs: INodeParams[] + + constructor() { + this.label = 'Upstash Redis API' + this.name = 'upstashRedisApi' + this.version = 1.0 + this.inputs = [ + { + label: 'Upstash Redis REST URL', + name: 'upstashConnectionUrl', + type: 'string' + }, + { + label: 'Token', + name: 'upstashConnectionToken', + type: 'password' + } + ] + } +} + +module.exports = { credClass: UpstashRedisApi } diff --git a/packages/components/nodes/cache/MomentoCache/MomentoCache.ts b/packages/components/nodes/cache/MomentoCache/MomentoCache.ts new file mode 100644 index 00000000..9aa82e82 --- /dev/null +++ b/packages/components/nodes/cache/MomentoCache/MomentoCache.ts @@ -0,0 +1,57 @@ +import { getBaseClasses, getCredentialData, getCredentialParam, ICommonObject, INode, INodeData, INodeParams } from '../../../src' +import { MomentoCache as LangchainMomentoCache } from 'langchain/cache/momento' +import { CacheClient, Configurations, CredentialProvider } from '@gomomento/sdk' + +class MomentoCache implements INode { + label: string + name: string + version: number + description: string + type: string + icon: string + category: string + baseClasses: string[] + inputs: INodeParams[] + credential: INodeParams + + constructor() { + this.label = 'Momento Cache' + this.name = 'momentoCache' + this.version = 1.0 + this.type = 'MomentoCache' + this.icon = 'momento.png' + this.category = 'Cache' + this.baseClasses = [this.type, ...getBaseClasses(LangchainMomentoCache)] + this.credential = { + label: 'Connect Credential', + name: 'credential', + type: 'credential', + optional: true, + credentialNames: ['momentoCacheApi'] + } + this.inputs = [] + } + + async init(nodeData: INodeData, _: string, options: ICommonObject): Promise { + const credentialData = await getCredentialData(nodeData.credential ?? '', options) + const apiKey = getCredentialParam('momentoApiKey', credentialData, nodeData) + const cacheName = getCredentialParam('momentoCache', credentialData, nodeData) + + // See https://github.com/momentohq/client-sdk-javascript for connection options + const client = new CacheClient({ + configuration: Configurations.Laptop.v1(), + credentialProvider: CredentialProvider.fromString({ + apiKey: apiKey + }), + defaultTtlSeconds: 60 * 60 * 24 + }) + + let momentoCache = await LangchainMomentoCache.fromProps({ + client, + cacheName: cacheName + }) + return momentoCache + } +} + +module.exports = { nodeClass: MomentoCache } diff --git a/packages/components/nodes/cache/MomentoCache/momento.png b/packages/components/nodes/cache/MomentoCache/momento.png new file mode 100644 index 00000000..0f2b54b6 Binary files /dev/null and b/packages/components/nodes/cache/MomentoCache/momento.png differ diff --git a/packages/components/nodes/cache/RedisCache/RedisCache.ts b/packages/components/nodes/cache/RedisCache/RedisCache.ts new file mode 100644 index 00000000..c1b08be6 --- /dev/null +++ b/packages/components/nodes/cache/RedisCache/RedisCache.ts @@ -0,0 +1,52 @@ +import { getBaseClasses, getCredentialData, getCredentialParam, ICommonObject, INode, INodeData, INodeParams } from '../../../src' +import { RedisCache as LangchainRedisCache } from 'langchain/cache/ioredis' +import { Redis } from 'ioredis' + +class RedisCache implements INode { + label: string + name: string + version: number + description: string + type: string + icon: string + category: string + baseClasses: string[] + inputs: INodeParams[] + credential: INodeParams + + constructor() { + this.label = 'Redis Cache' + this.name = 'redisCache' + this.version = 1.0 + this.type = 'RedisCache' + this.icon = 'redis.svg' + this.category = 'Cache' + this.baseClasses = [this.type, ...getBaseClasses(LangchainRedisCache)] + this.credential = { + label: 'Connect Credential', + name: 'credential', + type: 'credential', + optional: true, + credentialNames: ['redisCacheApi'] + } + this.inputs = [] + } + + async init(nodeData: INodeData, _: string, options: ICommonObject): Promise { + const credentialData = await getCredentialData(nodeData.credential ?? '', options) + const username = getCredentialParam('redisCacheUser', credentialData, nodeData) + const password = getCredentialParam('redisCachePwd', credentialData, nodeData) + const portStr = getCredentialParam('redisCachePort', credentialData, nodeData) + const host = getCredentialParam('redisCacheHost', credentialData, nodeData) + + const client = new Redis({ + port: portStr ? parseInt(portStr) : 6379, + host, + username, + password + }) + return new LangchainRedisCache(client) + } +} + +module.exports = { nodeClass: RedisCache } diff --git a/packages/components/nodes/cache/RedisCache/redis.svg b/packages/components/nodes/cache/RedisCache/redis.svg new file mode 100644 index 00000000..90359069 --- /dev/null +++ b/packages/components/nodes/cache/RedisCache/redis.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/packages/components/nodes/cache/UpstashRedisCache/UpstashRedisCache.ts b/packages/components/nodes/cache/UpstashRedisCache/UpstashRedisCache.ts new file mode 100644 index 00000000..eb5a9e2f --- /dev/null +++ b/packages/components/nodes/cache/UpstashRedisCache/UpstashRedisCache.ts @@ -0,0 +1,49 @@ +import { getBaseClasses, getCredentialData, getCredentialParam, ICommonObject, INode, INodeData, INodeParams } from '../../../src' +import { UpstashRedisCache as LangchainUpstashRedisCache } from 'langchain/cache/upstash_redis' + +class UpstashRedisCache implements INode { + label: string + name: string + version: number + description: string + type: string + icon: string + category: string + baseClasses: string[] + inputs: INodeParams[] + credential: INodeParams + + constructor() { + this.label = 'Upstash Redis Cache' + this.name = 'upstashRedisCache' + this.version = 1.0 + this.type = 'UpstashRedisCache' + this.icon = 'upstash.png' + this.category = 'Cache' + this.baseClasses = [this.type, ...getBaseClasses(LangchainUpstashRedisCache)] + this.credential = { + label: 'Connect Credential', + name: 'credential', + type: 'credential', + optional: true, + credentialNames: ['upstashRedisApi'] + } + this.inputs = [] + } + + async init(nodeData: INodeData, _: string, options: ICommonObject): Promise { + const credentialData = await getCredentialData(nodeData.credential ?? '', options) + const upstashConnectionUrl = getCredentialParam('upstashConnectionUrl', credentialData, nodeData) + const upstashToken = getCredentialParam('upstashConnectionToken', credentialData, nodeData) + + const cache = new LangchainUpstashRedisCache({ + config: { + url: upstashConnectionUrl, + token: upstashToken + } + }) + return cache + } +} + +module.exports = { nodeClass: UpstashRedisCache } diff --git a/packages/components/nodes/cache/UpstashRedisCache/upstash.png b/packages/components/nodes/cache/UpstashRedisCache/upstash.png new file mode 100644 index 00000000..e27e02f4 Binary files /dev/null and b/packages/components/nodes/cache/UpstashRedisCache/upstash.png differ diff --git a/packages/components/nodes/chatmodels/AWSBedrock/AWSChatBedrock.ts b/packages/components/nodes/chatmodels/AWSBedrock/AWSChatBedrock.ts index f5fa8bba..16fbc8dc 100644 --- a/packages/components/nodes/chatmodels/AWSBedrock/AWSChatBedrock.ts +++ b/packages/components/nodes/chatmodels/AWSBedrock/AWSChatBedrock.ts @@ -2,6 +2,8 @@ import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Inter import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils' import { ChatBedrock } from 'langchain/chat_models/bedrock' import { BaseBedrockInput } from 'langchain/dist/util/bedrock' +import { BaseCache } from 'langchain/schema' +import { BaseLLMParams } from 'langchain/llms/base' /** * I had to run the following to build the component @@ -25,7 +27,7 @@ class AWSChatBedrock_ChatModels implements INode { constructor() { this.label = 'AWS Bedrock' this.name = 'awsChatBedrock' - this.version = 1.1 + this.version = 2.0 this.type = 'AWSChatBedrock' this.icon = 'awsBedrock.png' this.category = 'Chat Models' @@ -39,6 +41,12 @@ class AWSChatBedrock_ChatModels implements INode { optional: true } this.inputs = [ + { + label: 'Cache', + name: 'cache', + type: 'BaseCache', + optional: true + }, { label: 'Region', name: 'region', @@ -130,8 +138,9 @@ class AWSChatBedrock_ChatModels implements INode { const iModel = nodeData.inputs?.model as string const iTemperature = nodeData.inputs?.temperature as string const iMax_tokens_to_sample = nodeData.inputs?.max_tokens_to_sample as string + const cache = nodeData.inputs?.cache as BaseCache - const obj: BaseBedrockInput = { + const obj: BaseBedrockInput & BaseLLMParams = { region: iRegion, model: iModel, maxTokens: parseInt(iMax_tokens_to_sample, 10), @@ -157,6 +166,7 @@ class AWSChatBedrock_ChatModels implements INode { sessionToken: credentialApiSession } } + if (cache) obj.cache = cache const amazonBedrock = new ChatBedrock(obj) return amazonBedrock diff --git a/packages/components/nodes/chatmodels/AzureChatOpenAI/AzureChatOpenAI.ts b/packages/components/nodes/chatmodels/AzureChatOpenAI/AzureChatOpenAI.ts index 90f430f0..99e151e6 100644 --- a/packages/components/nodes/chatmodels/AzureChatOpenAI/AzureChatOpenAI.ts +++ b/packages/components/nodes/chatmodels/AzureChatOpenAI/AzureChatOpenAI.ts @@ -2,6 +2,8 @@ import { OpenAIBaseInput } from 'langchain/dist/types/openai-types' import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface' import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils' import { AzureOpenAIInput, ChatOpenAI } from 'langchain/chat_models/openai' +import { BaseCache } from 'langchain/schema' +import { BaseLLMParams } from 'langchain/llms/base' class AzureChatOpenAI_ChatModels implements INode { label: string @@ -18,7 +20,7 @@ class AzureChatOpenAI_ChatModels implements INode { constructor() { this.label = 'Azure ChatOpenAI' this.name = 'azureChatOpenAI' - this.version = 1.0 + this.version = 2.0 this.type = 'AzureChatOpenAI' this.icon = 'Azure.svg' this.category = 'Chat Models' @@ -31,6 +33,12 @@ class AzureChatOpenAI_ChatModels implements INode { credentialNames: ['azureOpenAIApi'] } this.inputs = [ + { + label: 'Cache', + name: 'cache', + type: 'BaseCache', + optional: true + }, { label: 'Model Name', name: 'modelName', @@ -107,6 +115,7 @@ class AzureChatOpenAI_ChatModels implements INode { const presencePenalty = nodeData.inputs?.presencePenalty as string const timeout = nodeData.inputs?.timeout as string const streaming = nodeData.inputs?.streaming as boolean + const cache = nodeData.inputs?.cache as BaseCache const credentialData = await getCredentialData(nodeData.credential ?? '', options) const azureOpenAIApiKey = getCredentialParam('azureOpenAIApiKey', credentialData, nodeData) @@ -114,7 +123,7 @@ class AzureChatOpenAI_ChatModels implements INode { const azureOpenAIApiDeploymentName = getCredentialParam('azureOpenAIApiDeploymentName', credentialData, nodeData) const azureOpenAIApiVersion = getCredentialParam('azureOpenAIApiVersion', credentialData, nodeData) - const obj: Partial & Partial = { + const obj: Partial & BaseLLMParams & Partial = { temperature: parseFloat(temperature), modelName, azureOpenAIApiKey, @@ -128,6 +137,7 @@ class AzureChatOpenAI_ChatModels implements INode { if (frequencyPenalty) obj.frequencyPenalty = parseFloat(frequencyPenalty) if (presencePenalty) obj.presencePenalty = parseFloat(presencePenalty) if (timeout) obj.timeout = parseInt(timeout, 10) + if (cache) obj.cache = cache const model = new ChatOpenAI(obj) return model diff --git a/packages/components/nodes/chatmodels/Bittensor/Bittensor.ts b/packages/components/nodes/chatmodels/Bittensor/Bittensor.ts index cfcac863..36b084e6 100644 --- a/packages/components/nodes/chatmodels/Bittensor/Bittensor.ts +++ b/packages/components/nodes/chatmodels/Bittensor/Bittensor.ts @@ -1,6 +1,7 @@ import { INode, INodeData, INodeParams } from '../../../src/Interface' import { getBaseClasses } from '../../../src/utils' import { NIBittensorChatModel, BittensorInput } from 'langchain/experimental/chat_models/bittensor' +import { BaseCache } from 'langchain/schema' class Bittensor_ChatModels implements INode { label: string @@ -16,13 +17,19 @@ class Bittensor_ChatModels implements INode { constructor() { this.label = 'NIBittensorChat' this.name = 'NIBittensorChatModel' - this.version = 1.0 + this.version = 2.0 this.type = 'BittensorChat' this.icon = 'logo.png' this.category = 'Chat Models' this.description = 'Wrapper around Bittensor subnet 1 large language models' this.baseClasses = [this.type, ...getBaseClasses(NIBittensorChatModel)] this.inputs = [ + { + label: 'Cache', + name: 'cache', + type: 'BaseCache', + optional: true + }, { label: 'System prompt', name: 'system_prompt', @@ -35,9 +42,13 @@ class Bittensor_ChatModels implements INode { async init(nodeData: INodeData, _: string): Promise { const system_prompt = nodeData.inputs?.system_prompt as string + const cache = nodeData.inputs?.cache as BaseCache + const obj: Partial = { systemPrompt: system_prompt } + if (cache) obj.cache = cache + const model = new NIBittensorChatModel(obj) return model } diff --git a/packages/components/nodes/chatmodels/ChatAnthropic/ChatAnthropic.ts b/packages/components/nodes/chatmodels/ChatAnthropic/ChatAnthropic.ts index 12a33d99..f16968b6 100644 --- a/packages/components/nodes/chatmodels/ChatAnthropic/ChatAnthropic.ts +++ b/packages/components/nodes/chatmodels/ChatAnthropic/ChatAnthropic.ts @@ -1,6 +1,8 @@ import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface' import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils' import { AnthropicInput, ChatAnthropic } from 'langchain/chat_models/anthropic' +import { BaseCache } from 'langchain/schema' +import { BaseLLMParams } from 'langchain/llms/base' class ChatAnthropic_ChatModels implements INode { label: string @@ -17,7 +19,7 @@ class ChatAnthropic_ChatModels implements INode { constructor() { this.label = 'ChatAnthropic' this.name = 'chatAnthropic' - this.version = 1.0 + this.version = 2.0 this.type = 'ChatAnthropic' this.icon = 'chatAnthropic.png' this.category = 'Chat Models' @@ -30,6 +32,12 @@ class ChatAnthropic_ChatModels implements INode { credentialNames: ['anthropicApi'] } this.inputs = [ + { + label: 'Cache', + name: 'cache', + type: 'BaseCache', + optional: true + }, { label: 'Model Name', name: 'modelName', @@ -135,11 +143,12 @@ class ChatAnthropic_ChatModels implements INode { const topP = nodeData.inputs?.topP as string const topK = nodeData.inputs?.topK as string const streaming = nodeData.inputs?.streaming as boolean + const cache = nodeData.inputs?.cache as BaseCache const credentialData = await getCredentialData(nodeData.credential ?? '', options) const anthropicApiKey = getCredentialParam('anthropicApiKey', credentialData, nodeData) - const obj: Partial & { anthropicApiKey?: string } = { + const obj: Partial & BaseLLMParams & { anthropicApiKey?: string } = { temperature: parseFloat(temperature), modelName, anthropicApiKey, @@ -149,6 +158,7 @@ class ChatAnthropic_ChatModels implements INode { if (maxTokensToSample) obj.maxTokensToSample = parseInt(maxTokensToSample, 10) if (topP) obj.topP = parseFloat(topP) if (topK) obj.topK = parseFloat(topK) + if (cache) obj.cache = cache const model = new ChatAnthropic(obj) return model diff --git a/packages/components/nodes/chatmodels/ChatGooglePaLM/ChatGooglePaLM.ts b/packages/components/nodes/chatmodels/ChatGooglePaLM/ChatGooglePaLM.ts index 642ddb5e..121406c7 100644 --- a/packages/components/nodes/chatmodels/ChatGooglePaLM/ChatGooglePaLM.ts +++ b/packages/components/nodes/chatmodels/ChatGooglePaLM/ChatGooglePaLM.ts @@ -1,6 +1,7 @@ import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface' import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils' import { ChatGooglePaLM, GooglePaLMChatInput } from 'langchain/chat_models/googlepalm' +import { BaseCache } from 'langchain/schema' class ChatGooglePaLM_ChatModels implements INode { label: string @@ -17,7 +18,7 @@ class ChatGooglePaLM_ChatModels implements INode { constructor() { this.label = 'ChatGooglePaLM' this.name = 'chatGooglePaLM' - this.version = 1.0 + this.version = 2.0 this.type = 'ChatGooglePaLM' this.icon = 'Google_PaLM_Logo.svg' this.category = 'Chat Models' @@ -30,6 +31,12 @@ class ChatGooglePaLM_ChatModels implements INode { credentialNames: ['googleMakerSuite'] } this.inputs = [ + { + label: 'Cache', + name: 'cache', + type: 'BaseCache', + optional: true + }, { label: 'Model Name', name: 'modelName', @@ -96,6 +103,7 @@ class ChatGooglePaLM_ChatModels implements INode { const temperature = nodeData.inputs?.temperature as string const topP = nodeData.inputs?.topP as string const topK = nodeData.inputs?.topK as string + const cache = nodeData.inputs?.cache as BaseCache const credentialData = await getCredentialData(nodeData.credential ?? '', options) const googleMakerSuiteKey = getCredentialParam('googleMakerSuiteKey', credentialData, nodeData) @@ -108,6 +116,7 @@ class ChatGooglePaLM_ChatModels implements INode { if (topP) obj.topP = parseFloat(topP) if (topK) obj.topK = parseFloat(topK) + if (cache) obj.cache = cache const model = new ChatGooglePaLM(obj) return model diff --git a/packages/components/nodes/chatmodels/ChatGoogleVertexAI/ChatGoogleVertexAI.ts b/packages/components/nodes/chatmodels/ChatGoogleVertexAI/ChatGoogleVertexAI.ts index 4cb206f5..6b070bd9 100644 --- a/packages/components/nodes/chatmodels/ChatGoogleVertexAI/ChatGoogleVertexAI.ts +++ b/packages/components/nodes/chatmodels/ChatGoogleVertexAI/ChatGoogleVertexAI.ts @@ -2,6 +2,7 @@ import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Inter import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils' import { ChatGoogleVertexAI, GoogleVertexAIChatInput } from 'langchain/chat_models/googlevertexai' import { GoogleAuthOptions } from 'google-auth-library' +import { BaseCache } from 'langchain/schema' class GoogleVertexAI_ChatModels implements INode { label: string @@ -18,7 +19,7 @@ class GoogleVertexAI_ChatModels implements INode { constructor() { this.label = 'ChatGoogleVertexAI' this.name = 'chatGoogleVertexAI' - this.version = 1.0 + this.version = 2.0 this.type = 'ChatGoogleVertexAI' this.icon = 'vertexai.svg' this.category = 'Chat Models' @@ -34,6 +35,12 @@ class GoogleVertexAI_ChatModels implements INode { 'Google Vertex AI credential. If you are using a GCP service like Cloud Run, or if you have installed default credentials on your local machine, you do not need to set this credential.' } this.inputs = [ + { + label: 'Cache', + name: 'cache', + type: 'BaseCache', + optional: true + }, { label: 'Model Name', name: 'modelName', @@ -113,6 +120,7 @@ class GoogleVertexAI_ChatModels implements INode { const modelName = nodeData.inputs?.modelName as string const maxOutputTokens = nodeData.inputs?.maxOutputTokens as string const topP = nodeData.inputs?.topP as string + const cache = nodeData.inputs?.cache as BaseCache const obj: GoogleVertexAIChatInput = { temperature: parseFloat(temperature), @@ -122,6 +130,7 @@ class GoogleVertexAI_ChatModels implements INode { if (maxOutputTokens) obj.maxOutputTokens = parseInt(maxOutputTokens, 10) if (topP) obj.topP = parseFloat(topP) + if (cache) obj.cache = cache const model = new ChatGoogleVertexAI(obj) return model diff --git a/packages/components/nodes/chatmodels/ChatHuggingFace/ChatHuggingFace.ts b/packages/components/nodes/chatmodels/ChatHuggingFace/ChatHuggingFace.ts index ee55c7bb..153c5d10 100644 --- a/packages/components/nodes/chatmodels/ChatHuggingFace/ChatHuggingFace.ts +++ b/packages/components/nodes/chatmodels/ChatHuggingFace/ChatHuggingFace.ts @@ -1,6 +1,7 @@ import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface' import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils' import { HFInput, HuggingFaceInference } from './core' +import { BaseCache } from 'langchain/schema' class ChatHuggingFace_ChatModels implements INode { label: string @@ -17,7 +18,7 @@ class ChatHuggingFace_ChatModels implements INode { constructor() { this.label = 'ChatHuggingFace' this.name = 'chatHuggingFace' - this.version = 1.0 + this.version = 2.0 this.type = 'ChatHuggingFace' this.icon = 'huggingface.png' this.category = 'Chat Models' @@ -30,6 +31,12 @@ class ChatHuggingFace_ChatModels implements INode { credentialNames: ['huggingFaceApi'] } this.inputs = [ + { + label: 'Cache', + name: 'cache', + type: 'BaseCache', + optional: true + }, { label: 'Model', name: 'model', @@ -102,6 +109,7 @@ class ChatHuggingFace_ChatModels implements INode { const hfTopK = nodeData.inputs?.hfTopK as string const frequencyPenalty = nodeData.inputs?.frequencyPenalty as string const endpoint = nodeData.inputs?.endpoint as string + const cache = nodeData.inputs?.cache as BaseCache const credentialData = await getCredentialData(nodeData.credential ?? '', options) const huggingFaceApiKey = getCredentialParam('huggingFaceApiKey', credentialData, nodeData) @@ -119,6 +127,7 @@ class ChatHuggingFace_ChatModels implements INode { if (endpoint) obj.endpoint = endpoint const huggingFace = new HuggingFaceInference(obj) + if (cache) huggingFace.cache = cache return huggingFace } } diff --git a/packages/components/nodes/chatmodels/ChatLocalAI/ChatLocalAI.ts b/packages/components/nodes/chatmodels/ChatLocalAI/ChatLocalAI.ts index a6ddfae4..18ed409b 100644 --- a/packages/components/nodes/chatmodels/ChatLocalAI/ChatLocalAI.ts +++ b/packages/components/nodes/chatmodels/ChatLocalAI/ChatLocalAI.ts @@ -2,6 +2,8 @@ import { INode, INodeData, INodeParams } from '../../../src/Interface' import { getBaseClasses } from '../../../src/utils' import { OpenAIChat } from 'langchain/llms/openai' import { OpenAIChatInput } from 'langchain/chat_models/openai' +import { BaseCache } from 'langchain/schema' +import { BaseLLMParams } from 'langchain/llms/base' class ChatLocalAI_ChatModels implements INode { label: string @@ -17,13 +19,19 @@ class ChatLocalAI_ChatModels implements INode { constructor() { this.label = 'ChatLocalAI' this.name = 'chatLocalAI' - this.version = 1.0 + this.version = 2.0 this.type = 'ChatLocalAI' this.icon = 'localai.png' this.category = 'Chat Models' this.description = 'Use local LLMs like llama.cpp, gpt4all using LocalAI' this.baseClasses = [this.type, 'BaseChatModel', ...getBaseClasses(OpenAIChat)] this.inputs = [ + { + label: 'Cache', + name: 'cache', + type: 'BaseCache', + optional: true + }, { label: 'Base Path', name: 'basePath', @@ -78,8 +86,9 @@ class ChatLocalAI_ChatModels implements INode { const topP = nodeData.inputs?.topP as string const timeout = nodeData.inputs?.timeout as string const basePath = nodeData.inputs?.basePath as string + const cache = nodeData.inputs?.cache as BaseCache - const obj: Partial & { openAIApiKey?: string } = { + const obj: Partial & BaseLLMParams & { openAIApiKey?: string } = { temperature: parseFloat(temperature), modelName, openAIApiKey: 'sk-' @@ -88,6 +97,7 @@ class ChatLocalAI_ChatModels implements INode { if (maxTokens) obj.maxTokens = parseInt(maxTokens, 10) if (topP) obj.topP = parseFloat(topP) if (timeout) obj.timeout = parseInt(timeout, 10) + if (cache) obj.cache = cache const model = new OpenAIChat(obj, { basePath }) diff --git a/packages/components/nodes/chatmodels/ChatOpenAI/ChatOpenAI.ts b/packages/components/nodes/chatmodels/ChatOpenAI/ChatOpenAI.ts index ca081ff4..f74bd642 100644 --- a/packages/components/nodes/chatmodels/ChatOpenAI/ChatOpenAI.ts +++ b/packages/components/nodes/chatmodels/ChatOpenAI/ChatOpenAI.ts @@ -1,6 +1,8 @@ import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface' import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils' import { ChatOpenAI, OpenAIChatInput } from 'langchain/chat_models/openai' +import { BaseCache } from 'langchain/schema' +import { BaseLLMParams } from 'langchain/llms/base' class ChatOpenAI_ChatModels implements INode { label: string @@ -17,7 +19,7 @@ class ChatOpenAI_ChatModels implements INode { constructor() { this.label = 'ChatOpenAI' this.name = 'chatOpenAI' - this.version = 1.0 + this.version = 2.0 this.type = 'ChatOpenAI' this.icon = 'openai.png' this.category = 'Chat Models' @@ -30,6 +32,12 @@ class ChatOpenAI_ChatModels implements INode { credentialNames: ['openAIApi'] } this.inputs = [ + { + label: 'Cache', + name: 'cache', + type: 'BaseCache', + optional: true + }, { label: 'Model Name', name: 'modelName', @@ -151,7 +159,9 @@ class ChatOpenAI_ChatModels implements INode { const credentialData = await getCredentialData(nodeData.credential ?? '', options) const openAIApiKey = getCredentialParam('openAIApiKey', credentialData, nodeData) - const obj: Partial & { openAIApiKey?: string } = { + const cache = nodeData.inputs?.cache as BaseCache + + const obj: Partial & BaseLLMParams & { openAIApiKey?: string } = { temperature: parseFloat(temperature), modelName, openAIApiKey, @@ -163,6 +173,7 @@ class ChatOpenAI_ChatModels implements INode { if (frequencyPenalty) obj.frequencyPenalty = parseFloat(frequencyPenalty) if (presencePenalty) obj.presencePenalty = parseFloat(presencePenalty) if (timeout) obj.timeout = parseInt(timeout, 10) + if (cache) obj.cache = cache let parsedBaseOptions: any | undefined = undefined diff --git a/packages/components/nodes/chatmodels/ChatOpenAICustom/ChatOpenAICustom.ts b/packages/components/nodes/chatmodels/ChatOpenAICustom/ChatOpenAICustom.ts index 29c1181a..2a01a2e5 100644 --- a/packages/components/nodes/chatmodels/ChatOpenAICustom/ChatOpenAICustom.ts +++ b/packages/components/nodes/chatmodels/ChatOpenAICustom/ChatOpenAICustom.ts @@ -1,6 +1,8 @@ import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface' import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils' import { ChatOpenAI, OpenAIChatInput } from 'langchain/chat_models/openai' +import { BaseCache } from 'langchain/schema' +import { BaseLLMParams } from 'langchain/llms/base' class ChatOpenAICustom_ChatModels implements INode { label: string @@ -17,7 +19,7 @@ class ChatOpenAICustom_ChatModels implements INode { constructor() { this.label = 'ChatOpenAI Custom' this.name = 'chatOpenAICustom' - this.version = 1.0 + this.version = 2.0 this.type = 'ChatOpenAI-Custom' this.icon = 'openai.png' this.category = 'Chat Models' @@ -31,6 +33,12 @@ class ChatOpenAICustom_ChatModels implements INode { optional: true } this.inputs = [ + { + label: 'Cache', + name: 'cache', + type: 'BaseCache', + optional: true + }, { label: 'Model Name', name: 'modelName', @@ -113,11 +121,12 @@ class ChatOpenAICustom_ChatModels implements INode { const streaming = nodeData.inputs?.streaming as boolean const basePath = nodeData.inputs?.basepath as string const baseOptions = nodeData.inputs?.baseOptions + const cache = nodeData.inputs?.cache as BaseCache const credentialData = await getCredentialData(nodeData.credential ?? '', options) const openAIApiKey = getCredentialParam('openAIApiKey', credentialData, nodeData) - const obj: Partial & { openAIApiKey?: string } = { + const obj: Partial & BaseLLMParams & { openAIApiKey?: string } = { temperature: parseFloat(temperature), modelName, openAIApiKey, @@ -129,6 +138,7 @@ class ChatOpenAICustom_ChatModels implements INode { if (frequencyPenalty) obj.frequencyPenalty = parseFloat(frequencyPenalty) if (presencePenalty) obj.presencePenalty = parseFloat(presencePenalty) if (timeout) obj.timeout = parseInt(timeout, 10) + if (cache) obj.cache = cache let parsedBaseOptions: any | undefined = undefined diff --git a/packages/components/nodes/llms/AWSBedrock/AWSBedrock.ts b/packages/components/nodes/llms/AWSBedrock/AWSBedrock.ts index 68ee7ed2..8f57daac 100644 --- a/packages/components/nodes/llms/AWSBedrock/AWSBedrock.ts +++ b/packages/components/nodes/llms/AWSBedrock/AWSBedrock.ts @@ -2,6 +2,8 @@ import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Inter import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils' import { Bedrock } from 'langchain/llms/bedrock' import { BaseBedrockInput } from 'langchain/dist/util/bedrock' +import { BaseCache } from 'langchain/schema' +import { BaseLLMParams } from 'langchain/llms/base' /** * I had to run the following to build the component @@ -39,6 +41,12 @@ class AWSBedrock_LLMs implements INode { optional: true } this.inputs = [ + { + label: 'Cache', + name: 'cache', + type: 'BaseCache', + optional: true + }, { label: 'Region', name: 'region', @@ -130,8 +138,8 @@ class AWSBedrock_LLMs implements INode { const iModel = nodeData.inputs?.model as string const iTemperature = nodeData.inputs?.temperature as string const iMax_tokens_to_sample = nodeData.inputs?.max_tokens_to_sample as string - - const obj: Partial = { + const cache = nodeData.inputs?.cache as BaseCache + const obj: Partial & BaseLLMParams = { model: iModel, region: iRegion, temperature: parseFloat(iTemperature), @@ -157,6 +165,7 @@ class AWSBedrock_LLMs implements INode { sessionToken: credentialApiSession } } + if (cache) obj.cache = cache const amazonBedrock = new Bedrock(obj) return amazonBedrock diff --git a/packages/components/nodes/llms/Azure OpenAI/AzureOpenAI.ts b/packages/components/nodes/llms/Azure OpenAI/AzureOpenAI.ts index f48c4642..f50e3d95 100644 --- a/packages/components/nodes/llms/Azure OpenAI/AzureOpenAI.ts +++ b/packages/components/nodes/llms/Azure OpenAI/AzureOpenAI.ts @@ -1,7 +1,8 @@ import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface' import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils' import { AzureOpenAIInput, OpenAI, OpenAIInput } from 'langchain/llms/openai' - +import { BaseCache } from 'langchain/schema' +import { BaseLLMParams } from 'langchain/llms/base' class AzureOpenAI_LLMs implements INode { label: string name: string @@ -17,7 +18,7 @@ class AzureOpenAI_LLMs implements INode { constructor() { this.label = 'Azure OpenAI' this.name = 'azureOpenAI' - this.version = 1.0 + this.version = 2.0 this.type = 'AzureOpenAI' this.icon = 'Azure.svg' this.category = 'LLMs' @@ -30,6 +31,12 @@ class AzureOpenAI_LLMs implements INode { credentialNames: ['azureOpenAIApi'] } this.inputs = [ + { + label: 'Cache', + name: 'cache', + type: 'BaseCache', + optional: true + }, { label: 'Model Name', name: 'modelName', @@ -163,7 +170,9 @@ class AzureOpenAI_LLMs implements INode { const azureOpenAIApiDeploymentName = getCredentialParam('azureOpenAIApiDeploymentName', credentialData, nodeData) const azureOpenAIApiVersion = getCredentialParam('azureOpenAIApiVersion', credentialData, nodeData) - const obj: Partial & Partial = { + const cache = nodeData.inputs?.cache as BaseCache + + const obj: Partial & BaseLLMParams & Partial = { temperature: parseFloat(temperature), modelName, azureOpenAIApiKey, @@ -179,6 +188,7 @@ class AzureOpenAI_LLMs implements INode { if (presencePenalty) obj.presencePenalty = parseFloat(presencePenalty) if (timeout) obj.timeout = parseInt(timeout, 10) if (bestOf) obj.bestOf = parseInt(bestOf, 10) + if (cache) obj.cache = cache const model = new OpenAI(obj) return model diff --git a/packages/components/nodes/llms/Bittensor/Bittensor.ts b/packages/components/nodes/llms/Bittensor/Bittensor.ts index a87a7e48..e6cc2bb6 100644 --- a/packages/components/nodes/llms/Bittensor/Bittensor.ts +++ b/packages/components/nodes/llms/Bittensor/Bittensor.ts @@ -1,6 +1,8 @@ import { INode, INodeData, INodeParams } from '../../../src/Interface' import { getBaseClasses } from '../../../src/utils' import { NIBittensorLLM, BittensorInput } from 'langchain/experimental/llms/bittensor' +import { BaseCache } from 'langchain/schema' +import { BaseLLMParams } from 'langchain/llms/base' class Bittensor_LLMs implements INode { label: string @@ -16,13 +18,19 @@ class Bittensor_LLMs implements INode { constructor() { this.label = 'NIBittensorLLM' this.name = 'NIBittensorLLM' - this.version = 1.0 + this.version = 2.0 this.type = 'Bittensor' this.icon = 'logo.png' this.category = 'LLMs' this.description = 'Wrapper around Bittensor subnet 1 large language models' this.baseClasses = [this.type, ...getBaseClasses(NIBittensorLLM)] this.inputs = [ + { + label: 'Cache', + name: 'cache', + type: 'BaseCache', + optional: true + }, { label: 'System prompt', name: 'system_prompt', @@ -44,10 +52,13 @@ class Bittensor_LLMs implements INode { async init(nodeData: INodeData, _: string): Promise { const system_prompt = nodeData.inputs?.system_prompt as string const topResponses = Number(nodeData.inputs?.topResponses as number) - const obj: Partial = { + const cache = nodeData.inputs?.cache as BaseCache + + const obj: Partial & BaseLLMParams = { systemPrompt: system_prompt, topResponses: topResponses } + if (cache) obj.cache = cache const model = new NIBittensorLLM(obj) return model diff --git a/packages/components/nodes/llms/Cohere/Cohere.ts b/packages/components/nodes/llms/Cohere/Cohere.ts index 4a3a8a80..3fde0af0 100644 --- a/packages/components/nodes/llms/Cohere/Cohere.ts +++ b/packages/components/nodes/llms/Cohere/Cohere.ts @@ -1,6 +1,7 @@ import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface' import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils' import { Cohere, CohereInput } from './core' +import { BaseCache } from 'langchain/schema' class Cohere_LLMs implements INode { label: string @@ -17,7 +18,7 @@ class Cohere_LLMs implements INode { constructor() { this.label = 'Cohere' this.name = 'cohere' - this.version = 1.0 + this.version = 2.0 this.type = 'Cohere' this.icon = 'cohere.png' this.category = 'LLMs' @@ -30,6 +31,12 @@ class Cohere_LLMs implements INode { credentialNames: ['cohereApi'] } this.inputs = [ + { + label: 'Cache', + name: 'cache', + type: 'BaseCache', + optional: true + }, { label: 'Model Name', name: 'modelName', @@ -85,7 +92,7 @@ class Cohere_LLMs implements INode { const temperature = nodeData.inputs?.temperature as string const modelName = nodeData.inputs?.modelName as string const maxTokens = nodeData.inputs?.maxTokens as string - + const cache = nodeData.inputs?.cache as BaseCache const credentialData = await getCredentialData(nodeData.credential ?? '', options) const cohereApiKey = getCredentialParam('cohereApiKey', credentialData, nodeData) @@ -96,7 +103,7 @@ class Cohere_LLMs implements INode { if (maxTokens) obj.maxTokens = parseInt(maxTokens, 10) if (modelName) obj.model = modelName if (temperature) obj.temperature = parseFloat(temperature) - + if (cache) obj.cache = cache const model = new Cohere(obj) return model } diff --git a/packages/components/nodes/llms/GooglePaLM/GooglePaLM.ts b/packages/components/nodes/llms/GooglePaLM/GooglePaLM.ts index 24630360..d3212a1c 100644 --- a/packages/components/nodes/llms/GooglePaLM/GooglePaLM.ts +++ b/packages/components/nodes/llms/GooglePaLM/GooglePaLM.ts @@ -1,7 +1,7 @@ import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface' import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils' import { GooglePaLM, GooglePaLMTextInput } from 'langchain/llms/googlepalm' - +import { BaseCache } from 'langchain/schema' class GooglePaLM_LLMs implements INode { label: string name: string @@ -17,7 +17,7 @@ class GooglePaLM_LLMs implements INode { constructor() { this.label = 'GooglePaLM' this.name = 'GooglePaLM' - this.version = 1.0 + this.version = 2.0 this.type = 'GooglePaLM' this.icon = 'Google_PaLM_Logo.svg' this.category = 'LLMs' @@ -30,6 +30,12 @@ class GooglePaLM_LLMs implements INode { credentialNames: ['googleMakerSuite'] } this.inputs = [ + { + label: 'Cache', + name: 'cache', + type: 'BaseCache', + optional: true + }, { label: 'Model Name', name: 'modelName', @@ -126,6 +132,7 @@ class GooglePaLM_LLMs implements INode { const topP = nodeData.inputs?.topP as string const topK = nodeData.inputs?.topK as string const stopSequencesObj = nodeData.inputs?.stopSequencesObj + const cache = nodeData.inputs?.cache as BaseCache const credentialData = await getCredentialData(nodeData.credential ?? '', options) const googleMakerSuiteKey = getCredentialParam('googleMakerSuiteKey', credentialData, nodeData) @@ -139,6 +146,7 @@ class GooglePaLM_LLMs implements INode { if (maxOutputTokens) obj.maxOutputTokens = parseInt(maxOutputTokens, 10) if (topP) obj.topP = parseFloat(topP) if (topK) obj.topK = parseFloat(topK) + if (cache) obj.cache = cache let parsedStopSequences: any | undefined = undefined if (stopSequencesObj) { diff --git a/packages/components/nodes/llms/GoogleVertexAI/GoogleVertexAI.ts b/packages/components/nodes/llms/GoogleVertexAI/GoogleVertexAI.ts index 4d19d04f..6b6d534b 100644 --- a/packages/components/nodes/llms/GoogleVertexAI/GoogleVertexAI.ts +++ b/packages/components/nodes/llms/GoogleVertexAI/GoogleVertexAI.ts @@ -2,6 +2,7 @@ import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Inter import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils' import { GoogleVertexAI, GoogleVertexAITextInput } from 'langchain/llms/googlevertexai' import { GoogleAuthOptions } from 'google-auth-library' +import { BaseCache } from 'langchain/schema' class GoogleVertexAI_LLMs implements INode { label: string @@ -18,7 +19,7 @@ class GoogleVertexAI_LLMs implements INode { constructor() { this.label = 'GoogleVertexAI' this.name = 'googlevertexai' - this.version = 1.0 + this.version = 2.0 this.type = 'GoogleVertexAI' this.icon = 'vertexai.svg' this.category = 'LLMs' @@ -34,6 +35,12 @@ class GoogleVertexAI_LLMs implements INode { 'Google Vertex AI credential. If you are using a GCP service like Cloud Run, or if you have installed default credentials on your local machine, you do not need to set this credential.' } this.inputs = [ + { + label: 'Cache', + name: 'cache', + type: 'BaseCache', + optional: true + }, { label: 'Model Name', name: 'modelName', @@ -120,6 +127,7 @@ class GoogleVertexAI_LLMs implements INode { const modelName = nodeData.inputs?.modelName as string const maxOutputTokens = nodeData.inputs?.maxOutputTokens as string const topP = nodeData.inputs?.topP as string + const cache = nodeData.inputs?.cache as BaseCache const obj: Partial = { temperature: parseFloat(temperature), @@ -129,6 +137,7 @@ class GoogleVertexAI_LLMs implements INode { if (maxOutputTokens) obj.maxOutputTokens = parseInt(maxOutputTokens, 10) if (topP) obj.topP = parseFloat(topP) + if (cache) obj.cache = cache const model = new GoogleVertexAI(obj) return model diff --git a/packages/components/nodes/llms/HuggingFaceInference/HuggingFaceInference.ts b/packages/components/nodes/llms/HuggingFaceInference/HuggingFaceInference.ts index c7f6a37e..8dcf021b 100644 --- a/packages/components/nodes/llms/HuggingFaceInference/HuggingFaceInference.ts +++ b/packages/components/nodes/llms/HuggingFaceInference/HuggingFaceInference.ts @@ -1,6 +1,7 @@ import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface' import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils' import { HFInput, HuggingFaceInference } from './core' +import { BaseCache } from 'langchain/schema' class HuggingFaceInference_LLMs implements INode { label: string @@ -17,7 +18,7 @@ class HuggingFaceInference_LLMs implements INode { constructor() { this.label = 'HuggingFace Inference' this.name = 'huggingFaceInference_LLMs' - this.version = 1.0 + this.version = 2.0 this.type = 'HuggingFaceInference' this.icon = 'huggingface.png' this.category = 'LLMs' @@ -30,6 +31,12 @@ class HuggingFaceInference_LLMs implements INode { credentialNames: ['huggingFaceApi'] } this.inputs = [ + { + label: 'Cache', + name: 'cache', + type: 'BaseCache', + optional: true + }, { label: 'Model', name: 'model', @@ -106,6 +113,8 @@ class HuggingFaceInference_LLMs implements INode { const credentialData = await getCredentialData(nodeData.credential ?? '', options) const huggingFaceApiKey = getCredentialParam('huggingFaceApiKey', credentialData, nodeData) + const cache = nodeData.inputs?.cache as BaseCache + const obj: Partial = { model, apiKey: huggingFaceApiKey @@ -119,6 +128,8 @@ class HuggingFaceInference_LLMs implements INode { if (endpoint) obj.endpoint = endpoint const huggingFace = new HuggingFaceInference(obj) + if (cache) huggingFace.cache = cache + return huggingFace } } diff --git a/packages/components/nodes/llms/OpenAI/OpenAI.ts b/packages/components/nodes/llms/OpenAI/OpenAI.ts index 2960ad2a..9109dd40 100644 --- a/packages/components/nodes/llms/OpenAI/OpenAI.ts +++ b/packages/components/nodes/llms/OpenAI/OpenAI.ts @@ -1,6 +1,8 @@ import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface' import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils' import { OpenAI, OpenAIInput } from 'langchain/llms/openai' +import { BaseLLMParams } from 'langchain/llms/base' +import { BaseCache } from 'langchain/schema' class OpenAI_LLMs implements INode { label: string @@ -17,7 +19,7 @@ class OpenAI_LLMs implements INode { constructor() { this.label = 'OpenAI' this.name = 'openAI' - this.version = 2.0 + this.version = 3.0 this.type = 'OpenAI' this.icon = 'openai.png' this.category = 'LLMs' @@ -30,6 +32,12 @@ class OpenAI_LLMs implements INode { credentialNames: ['openAIApi'] } this.inputs = [ + { + label: 'Cache', + name: 'cache', + type: 'BaseCache', + optional: true + }, { label: 'Model Name', name: 'modelName', @@ -149,7 +157,9 @@ class OpenAI_LLMs implements INode { const credentialData = await getCredentialData(nodeData.credential ?? '', options) const openAIApiKey = getCredentialParam('openAIApiKey', credentialData, nodeData) - const obj: Partial & { openAIApiKey?: string } = { + const cache = nodeData.inputs?.cache as BaseCache + + const obj: Partial & BaseLLMParams & { openAIApiKey?: string } = { temperature: parseFloat(temperature), modelName, openAIApiKey, @@ -164,8 +174,9 @@ class OpenAI_LLMs implements INode { if (batchSize) obj.batchSize = parseInt(batchSize, 10) if (bestOf) obj.bestOf = parseInt(bestOf, 10) - let parsedBaseOptions: any | undefined = undefined + if (cache) obj.cache = cache + let parsedBaseOptions: any | undefined = undefined if (baseOptions) { try { parsedBaseOptions = typeof baseOptions === 'object' ? baseOptions : JSON.parse(baseOptions) diff --git a/packages/components/nodes/llms/Replicate/Replicate.ts b/packages/components/nodes/llms/Replicate/Replicate.ts index 22c6e93a..fd5373a1 100644 --- a/packages/components/nodes/llms/Replicate/Replicate.ts +++ b/packages/components/nodes/llms/Replicate/Replicate.ts @@ -1,6 +1,8 @@ import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface' import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils' import { Replicate, ReplicateInput } from 'langchain/llms/replicate' +import { BaseCache } from 'langchain/schema' +import { BaseLLMParams } from 'langchain/llms/base' class Replicate_LLMs implements INode { label: string @@ -17,7 +19,7 @@ class Replicate_LLMs implements INode { constructor() { this.label = 'Replicate' this.name = 'replicate' - this.version = 1.0 + this.version = 2.0 this.type = 'Replicate' this.icon = 'replicate.svg' this.category = 'LLMs' @@ -30,6 +32,12 @@ class Replicate_LLMs implements INode { credentialNames: ['replicateApi'] } this.inputs = [ + { + label: 'Cache', + name: 'cache', + type: 'BaseCache', + optional: true + }, { label: 'Model', name: 'model', @@ -103,7 +111,9 @@ class Replicate_LLMs implements INode { const name = modelName.split(':')[0].split('/').pop() const org = modelName.split(':')[0].split('/')[0] - const obj: ReplicateInput = { + const cache = nodeData.inputs?.cache as BaseCache + + const obj: ReplicateInput & BaseLLMParams = { model: `${org}/${name}:${version}`, apiKey } @@ -120,6 +130,8 @@ class Replicate_LLMs implements INode { } if (Object.keys(inputs).length) obj.input = inputs + if (cache) obj.cache = cache + const model = new Replicate(obj) return model } diff --git a/packages/components/package.json b/packages/components/package.json index a3bd992c..e3c4380a 100644 --- a/packages/components/package.json +++ b/packages/components/package.json @@ -19,6 +19,7 @@ "@aws-sdk/client-dynamodb": "^3.360.0", "@dqbd/tiktoken": "^1.0.7", "@getzep/zep-js": "^0.6.3", + "@gomomento/sdk": "^1.40.2", "@google-ai/generativelanguage": "^0.2.1", "@huggingface/inference": "^2.6.1", "@notionhq/client": "^2.2.8", @@ -43,6 +44,7 @@ "google-auth-library": "^9.0.0", "graphql": "^16.6.0", "html-to-text": "^9.0.5", + "ioredis": "^5.3.2", "langchain": "^0.0.157", "langfuse-langchain": "^1.0.14-alpha.0", "langsmith": "^0.0.32", diff --git a/packages/components/src/handler.ts b/packages/components/src/handler.ts index 10f9a214..3b1952d6 100644 --- a/packages/components/src/handler.ts +++ b/packages/components/src/handler.ts @@ -151,6 +151,7 @@ export class CustomChainHandler extends BaseCallbackHandler { socketIOClientId = '' skipK = 0 // Skip streaming for first K numbers of handleLLMStart returnSourceDocuments = false + cachedResponse = true constructor(socketIO: Server, socketIOClientId: string, skipK?: number, returnSourceDocuments?: boolean) { super() @@ -161,6 +162,7 @@ export class CustomChainHandler extends BaseCallbackHandler { } handleLLMStart() { + this.cachedResponse = false if (this.skipK > 0) this.skipK -= 1 } @@ -178,9 +180,30 @@ export class CustomChainHandler extends BaseCallbackHandler { this.socketIO.to(this.socketIOClientId).emit('end') } - handleChainEnd(outputs: ChainValues): void | Promise { - if (this.returnSourceDocuments) { - this.socketIO.to(this.socketIOClientId).emit('sourceDocuments', outputs?.sourceDocuments) + handleChainEnd(outputs: ChainValues, _: string, parentRunId?: string): void | Promise { + /* + Langchain does not call handleLLMStart, handleLLMEnd, handleLLMNewToken when the chain is cached. + Callback Order is "Chain Start -> LLM Start --> LLM Token --> LLM End -> Chain End" for normal responses. + Callback Order is "Chain Start -> Chain End" for cached responses. + */ + if (this.cachedResponse && parentRunId === undefined) { + const cachedValue = outputs.text ?? outputs.response ?? outputs.output ?? outputs.output_text + //split at whitespace, and keep the whitespace. This is to preserve the original formatting. + const result = cachedValue.split(/(\s+)/) + result.forEach((token: string, index: number) => { + if (index === 0) { + this.socketIO.to(this.socketIOClientId).emit('start', token) + } + this.socketIO.to(this.socketIOClientId).emit('token', token) + }) + if (this.returnSourceDocuments) { + this.socketIO.to(this.socketIOClientId).emit('sourceDocuments', outputs?.sourceDocuments) + } + this.socketIO.to(this.socketIOClientId).emit('end') + } else { + if (this.returnSourceDocuments) { + this.socketIO.to(this.socketIOClientId).emit('sourceDocuments', outputs?.sourceDocuments) + } } } } diff --git a/packages/server/src/utils/index.ts b/packages/server/src/utils/index.ts index bb048490..664f110b 100644 --- a/packages/server/src/utils/index.ts +++ b/packages/server/src/utils/index.ts @@ -477,6 +477,7 @@ export const replaceInputsWithConfig = (flowNodeData: INodeData, overrideConfig: */ export const isStartNodeDependOnInput = (startingNodes: IReactFlowNode[], nodes: IReactFlowNode[]): boolean => { for (const node of startingNodes) { + if (node.data.category === 'Cache') return true for (const inputName in node.data.inputs) { const inputVariables = getInputVariables(node.data.inputs[inputName]) if (inputVariables.length > 0) return true