Addition of Claude for Image uploads

packages/components/nodes/chatmodels/ChatAnthropic/ChatAnthropic.ts

@@ -1,8 +1,9 @@
-import { AnthropicInput, ChatAnthropic } from '@langchain/anthropic'
+import { AnthropicInput, ChatAnthropic as LangchainChatAnthropic } from '@langchain/anthropic'
 import { BaseCache } from '@langchain/core/caches'
 import { BaseLLMParams } from '@langchain/core/language_models/llms'
-import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
+import { ICommonObject, IMultiModalOption, INode, INodeData, INodeParams } from '../../../src/Interface'
 import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
+import { ChatAnthropic } from './FlowiseChatAntrhopic'
 
 class ChatAnthropic_ChatModels implements INode {
     label: string
@@ -19,12 +20,12 @@ class ChatAnthropic_ChatModels implements INode {
     constructor() {
         this.label = 'ChatAnthropic'
         this.name = 'chatAnthropic'
-        this.version = 3.0
+        this.version = 4.0
         this.type = 'ChatAnthropic'
         this.icon = 'Anthropic.svg'
         this.category = 'Chat Models'
         this.description = 'Wrapper around ChatAnthropic large language models that use the Chat endpoint'
-        this.baseClasses = [this.type, ...getBaseClasses(ChatAnthropic)]
+        this.baseClasses = [this.type, ...getBaseClasses(LangchainChatAnthropic)]
         this.credential = {
             label: 'Connect Credential',
             name: 'credential',
@@ -147,6 +148,15 @@ class ChatAnthropic_ChatModels implements INode {
                 step: 0.1,
                 optional: true,
                 additionalParams: true
+            },
+            {
+                label: 'Allow Image Uploads',
+                name: 'allowImageUploads',
+                type: 'boolean',
+                description:
+                    'Automatically uses claude-3-* models when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent',
+                default: false,
+                optional: true
             }
         ]
     }
@@ -163,6 +173,8 @@ class ChatAnthropic_ChatModels implements INode {
         const credentialData = await getCredentialData(nodeData.credential ?? '', options)
         const anthropicApiKey = getCredentialParam('anthropicApiKey', credentialData, nodeData)
 
+        const allowImageUploads = nodeData.inputs?.allowImageUploads as boolean
+
         const obj: Partial<AnthropicInput> & BaseLLMParams & { anthropicApiKey?: string } = {
             temperature: parseFloat(temperature),
             modelName,
@@ -175,7 +187,14 @@ class ChatAnthropic_ChatModels implements INode {
         if (topK) obj.topK = parseFloat(topK)
         if (cache) obj.cache = cache
 
-        const model = new ChatAnthropic(obj)
+        const multiModalOption: IMultiModalOption = {
+            image: {
+                allowImageUploads: allowImageUploads ?? false
+            }
+        }
+
+        const model = new ChatAnthropic(nodeData.id, obj)
+        model.setMultiModalOption(multiModalOption)
         return model
     }
 }

packages/components/nodes/chatmodels/ChatAnthropic/FlowiseChatAntrhopic.ts

@@ -0,0 +1,34 @@
+import { AnthropicInput, ChatAnthropic as LangchainChatAnthropic } from '@langchain/anthropic'
+import { IMultiModalOption } from '../../../src'
+import { IVisionChatModal } from '../../../src/IVisionChatModal'
+import { BaseLLMParams } from '@langchain/core/language_models/llms'
+
+export class ChatAnthropic extends LangchainChatAnthropic implements IVisionChatModal {
+    configuredModel: string
+    configuredMaxToken: number
+    multiModalOption: IMultiModalOption
+    id: string
+
+    constructor(id: string, fields: Partial<AnthropicInput> & BaseLLMParams & { anthropicApiKey?: string }) {
+        super(fields)
+        this.id = id
+        this.configuredModel = fields?.modelName || 'claude-3-opus-20240229'
+        this.configuredMaxToken = fields?.maxTokens ?? 256
+    }
+
+    revertToOriginalModel(): void {
+        super.modelName = this.configuredModel
+        super.maxTokens = this.configuredMaxToken
+    }
+
+    setMultiModalOption(multiModalOption: IMultiModalOption): void {
+        this.multiModalOption = multiModalOption
+    }
+
+    setVisionModel(): void {
+        if (!this.modelName.startsWith('claude-3')) {
+            super.modelName = 'claude-3-opus-20240229'
+            super.maxTokens = 1024
+        }
+    }
+}

packages/components/nodes/chatmodels/ChatOpenAI/ChatOpenAI.ts

@@ -228,7 +228,7 @@ class ChatOpenAI_ChatModels implements INode {
 
         const obj: Partial<OpenAIChatInput> &
             Partial<AzureOpenAIInput> &
-            BaseChatModelParams & { configuration?: ClientOptions & LegacyOpenAIInput; multiModalOption?: IMultiModalOption } = {
+            BaseChatModelParams & { configuration?: ClientOptions & LegacyOpenAIInput } = {
             temperature: parseFloat(temperature),
             modelName,
             openAIApiKey,
@@ -265,10 +265,9 @@ class ChatOpenAI_ChatModels implements INode {
                 imageResolution
             }
         }
-        obj.multiModalOption = multiModalOption
 
         const model = new ChatOpenAI(nodeData.id, obj)
-
+        model.setMultiModalOption(multiModalOption)
         return model
     }
 }

packages/components/nodes/chatmodels/ChatOpenAI/FlowiseChatOpenAI.ts

@@ -1,39 +1,40 @@
 import type { ClientOptions } from 'openai'
-import {
-    ChatOpenAI as LangchainChatOpenAI,
-    OpenAIChatInput,
-    LegacyOpenAIInput,
-    AzureOpenAIInput,
-    ChatOpenAICallOptions
-} from '@langchain/openai'
+import { ChatOpenAI as LangchainChatOpenAI, OpenAIChatInput, LegacyOpenAIInput, AzureOpenAIInput } from '@langchain/openai'
 import { BaseChatModelParams } from '@langchain/core/language_models/chat_models'
-import { BaseMessageLike } from '@langchain/core/messages'
-import { Callbacks } from '@langchain/core/callbacks/manager'
-import { LLMResult } from '@langchain/core/outputs'
 import { IMultiModalOption } from '../../../src'
+import { IVisionChatModal } from '../../../src/IVisionChatModal'
 
-export class ChatOpenAI extends LangchainChatOpenAI {
+export class ChatOpenAI extends LangchainChatOpenAI implements IVisionChatModal {
     configuredModel: string
-    configuredMaxToken?: number
-    multiModalOption?: IMultiModalOption
+    configuredMaxToken: number
+    multiModalOption: IMultiModalOption
     id: string
 
     constructor(
         id: string,
         fields?: Partial<OpenAIChatInput> &
             Partial<AzureOpenAIInput> &
-            BaseChatModelParams & { configuration?: ClientOptions & LegacyOpenAIInput; multiModalOption?: IMultiModalOption },
+            BaseChatModelParams & { configuration?: ClientOptions & LegacyOpenAIInput },
         /** @deprecated */
         configuration?: ClientOptions & LegacyOpenAIInput
     ) {
         super(fields, configuration)
         this.id = id
-        this.multiModalOption = fields?.multiModalOption
         this.configuredModel = fields?.modelName ?? 'gpt-3.5-turbo'
-        this.configuredMaxToken = fields?.maxTokens
+        this.configuredMaxToken = fields?.maxTokens ?? 256
     }
 
-    async generate(messages: BaseMessageLike[][], options?: string[] | ChatOpenAICallOptions, callbacks?: Callbacks): Promise<LLMResult> {
-        return super.generate(messages, options, callbacks)
+    revertToOriginalModel(): void {
+        super.modelName = this.configuredModel
+        super.maxTokens = this.configuredMaxToken
+    }
+
+    setMultiModalOption(multiModalOption: IMultiModalOption): void {
+        this.multiModalOption = multiModalOption
+    }
+
+    setVisionModel(): void {
+        super.modelName = 'gpt-4-vision-preview'
+        super.maxTokens = 1024
     }
 }