Merge branch 'main' into FEATURE/Vision

# Conflicts:
#	packages/components/nodes/agents/ConversationalAgent/ConversationalAgent.ts
#	packages/components/nodes/agents/MRKLAgentChat/MRKLAgentChat.ts
#	packages/components/nodes/chains/ConversationChain/ConversationChain.ts
#	packages/components/nodes/chains/LLMChain/LLMChain.ts
#	packages/components/nodes/chatmodels/ChatOpenAI/ChatOpenAI.ts

packages/components/nodes/chatmodels/AWSBedrock/AWSChatBedrock.ts

@@ -1,9 +1,9 @@
+import { BedrockChat } from '@langchain/community/chat_models/bedrock'
+import { BaseCache } from '@langchain/core/caches'
+import { BaseChatModelParams } from '@langchain/core/language_models/chat_models'
+import { BaseBedrockInput } from 'langchain/dist/util/bedrock'
 import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
 import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
-import { BedrockChat } from 'langchain/chat_models/bedrock'
-import { BaseBedrockInput } from 'langchain/dist/util/bedrock'
-import { BaseCache } from 'langchain/schema'
-import { BaseChatModelParams } from 'langchain/chat_models/base'
 
 /**
  * I had to run the following to build the component

packages/components/nodes/chatmodels/AzureChatOpenAI/AzureChatOpenAI.ts

@@ -1,8 +1,8 @@
+import { AzureOpenAIInput, ChatOpenAI, OpenAIChatInput } from '@langchain/openai'
+import { BaseCache } from '@langchain/core/caches'
+import { BaseLLMParams } from '@langchain/core/language_models/llms'
 import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
 import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
-import { AzureOpenAIInput, ChatOpenAI, OpenAIChatInput } from 'langchain/chat_models/openai'
-import { BaseCache } from 'langchain/schema'
-import { BaseLLMParams } from 'langchain/llms/base'
 
 class AzureChatOpenAI_ChatModels implements INode {
     label: string

packages/components/nodes/chatmodels/Bittensor/Bittensor.ts

@@ -1,7 +1,7 @@
+import { BaseCache } from '@langchain/core/caches'
+import { NIBittensorChatModel, BittensorInput } from 'langchain/experimental/chat_models/bittensor'
 import { INode, INodeData, INodeParams } from '../../../src/Interface'
 import { getBaseClasses } from '../../../src/utils'
-import { NIBittensorChatModel, BittensorInput } from 'langchain/experimental/chat_models/bittensor'
-import { BaseCache } from 'langchain/schema'
 
 class Bittensor_ChatModels implements INode {
     label: string

packages/components/nodes/chatmodels/ChatAnthropic/ChatAnthropic.ts

@@ -1,8 +1,8 @@
+import { AnthropicInput, ChatAnthropic } from '@langchain/anthropic'
+import { BaseCache } from '@langchain/core/caches'
+import { BaseLLMParams } from '@langchain/core/language_models/llms'
 import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
 import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
-import { AnthropicInput, ChatAnthropic } from 'langchain/chat_models/anthropic'
-import { BaseCache } from 'langchain/schema'
-import { BaseLLMParams } from 'langchain/llms/base'
 
 class ChatAnthropic_ChatModels implements INode {
     label: string

packages/components/nodes/chatmodels/ChatGoogleGenerativeAI/ChatGoogleGenerativeAI.ts

@@ -1,9 +1,9 @@
-import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
-import { convertMultiOptionsToStringArray, getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
-import { BaseCache } from 'langchain/schema'
-import { ChatGoogleGenerativeAI, GoogleGenerativeAIChatInput } from '@langchain/google-genai'
 import { HarmBlockThreshold, HarmCategory } from '@google/generative-ai'
 import type { SafetySetting } from '@google/generative-ai'
+import { ChatGoogleGenerativeAI, GoogleGenerativeAIChatInput } from '@langchain/google-genai'
+import { BaseCache } from '@langchain/core/caches'
+import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
+import { convertMultiOptionsToStringArray, getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
 
 class GoogleGenerativeAI_ChatModels implements INode {
     label: string

packages/components/nodes/chatmodels/ChatGooglePaLM/ChatGooglePaLM.ts

@@ -1,7 +1,7 @@
+import { ChatGooglePaLM, GooglePaLMChatInput } from '@langchain/community/chat_models/googlepalm'
+import { BaseCache } from '@langchain/core/caches'
 import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
 import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
-import { ChatGooglePaLM, GooglePaLMChatInput } from 'langchain/chat_models/googlepalm'
-import { BaseCache } from 'langchain/schema'
 
 class ChatGooglePaLM_ChatModels implements INode {
     label: string

packages/components/nodes/chatmodels/ChatGoogleVertexAI/ChatGoogleVertexAI.ts

@@ -1,8 +1,8 @@
+import { GoogleAuthOptions } from 'google-auth-library'
+import { BaseCache } from '@langchain/core/caches'
+import { ChatGoogleVertexAI, GoogleVertexAIChatInput } from '@langchain/community/chat_models/googlevertexai'
 import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
 import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
-import { ChatGoogleVertexAI, GoogleVertexAIChatInput } from 'langchain/chat_models/googlevertexai'
-import { GoogleAuthOptions } from 'google-auth-library'
-import { BaseCache } from 'langchain/schema'
 
 class GoogleVertexAI_ChatModels implements INode {
     label: string

packages/components/nodes/chatmodels/ChatHuggingFace/ChatHuggingFace.ts

@@ -1,7 +1,7 @@
+import { BaseCache } from '@langchain/core/caches'
+import { HFInput, HuggingFaceInference } from './core'
 import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
 import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
-import { HFInput, HuggingFaceInference } from './core'
-import { BaseCache } from 'langchain/schema'
 
 class ChatHuggingFace_ChatModels implements INode {
     label: string

packages/components/nodes/chatmodels/ChatHuggingFace/core.ts

@@ -1,5 +1,5 @@
+import { LLM, BaseLLMParams } from '@langchain/core/language_models/llms'
 import { getEnvironmentVariable } from '../../../src/utils'
-import { LLM, BaseLLMParams } from 'langchain/llms/base'
 
 export interface HFInput {
     /** Model to use */

packages/components/nodes/chatmodels/ChatLocalAI/ChatLocalAI.ts

@@ -1,9 +1,8 @@
+import { OpenAIChatInput, ChatOpenAI } from '@langchain/openai'
+import { BaseCache } from '@langchain/core/caches'
+import { BaseLLMParams } from '@langchain/core/language_models/llms'
 import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
 import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
-import { OpenAIChat } from 'langchain/llms/openai'
-import { OpenAIChatInput } from 'langchain/chat_models/openai'
-import { BaseCache } from 'langchain/schema'
-import { BaseLLMParams } from 'langchain/llms/base'
 
 class ChatLocalAI_ChatModels implements INode {
     label: string
@@ -25,7 +24,7 @@ class ChatLocalAI_ChatModels implements INode {
         this.icon = 'localai.png'
         this.category = 'Chat Models'
         this.description = 'Use local LLMs like llama.cpp, gpt4all using LocalAI'
-        this.baseClasses = [this.type, 'BaseChatModel', ...getBaseClasses(OpenAIChat)]
+        this.baseClasses = [this.type, 'BaseChatModel', ...getBaseClasses(ChatOpenAI)]
         this.credential = {
             label: 'Connect Credential',
             name: 'credential',
@@ -111,7 +110,7 @@ class ChatLocalAI_ChatModels implements INode {
         if (cache) obj.cache = cache
         if (localAIApiKey) obj.openAIApiKey = localAIApiKey
 
-        const model = new OpenAIChat(obj, { basePath })
+        const model = new ChatOpenAI(obj, { basePath })
 
         return model
     }

packages/components/nodes/chatmodels/ChatMistral/ChatMistral.ts

@@ -1,7 +1,7 @@
+import { BaseCache } from '@langchain/core/caches'
+import { ChatMistralAI, ChatMistralAIInput } from '@langchain/mistralai'
 import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
 import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
-import { BaseCache } from 'langchain/schema'
-import { ChatMistralAI, ChatMistralAIInput } from '@langchain/mistralai'
 
 class ChatMistral_ChatModels implements INode {
     label: string

packages/components/nodes/chatmodels/ChatOllama/ChatOllama.ts

@@ -1,8 +1,8 @@
+import { ChatOllama, ChatOllamaInput } from '@langchain/community/chat_models/ollama'
+import { BaseCache } from '@langchain/core/caches'
+import { BaseLLMParams } from '@langchain/core/language_models/llms'
 import { INode, INodeData, INodeParams } from '../../../src/Interface'
 import { getBaseClasses } from '../../../src/utils'
-import { ChatOllama, ChatOllamaInput } from 'langchain/chat_models/ollama'
-import { BaseCache } from 'langchain/schema'
-import { BaseLLMParams } from 'langchain/llms/base'
 
 class ChatOllama_ChatModels implements INode {
     label: string

packages/components/nodes/chatmodels/ChatOpenAI/ChatOpenAI.ts

@@ -1,8 +1,9 @@
+import type { ClientOptions } from 'openai'
+import { ChatOpenAI as LangchainChatOpenAI, OpenAIChatInput, AzureOpenAIInput, LegacyOpenAIInput } from '@langchain/openai'
+import { BaseCache } from '@langchain/core/caches'
+import { BaseChatModelParams } from '@langchain/core/language_models/chat_models'
 import { ICommonObject, IMultiModalOption, INode, INodeData, INodeParams } from '../../../src/Interface'
 import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
-import { ChatOpenAI as LangchainChatOpenAI, OpenAIChatInput } from 'langchain/chat_models/openai'
-import { BaseCache } from 'langchain/schema'
-import { BaseLLMParams } from 'langchain/llms/base'
 import { ChatOpenAI } from './FlowiseChatOpenAI'
 
 class ChatOpenAI_ChatModels implements INode {
@@ -60,6 +61,10 @@ class ChatOpenAI_ChatModels implements INode {
                         label: 'gpt-4-1106-preview',
                         name: 'gpt-4-1106-preview'
                     },
+                    {
+                        label: 'gpt-4-1106-vision-preview',
+                        name: 'gpt-4-1106-vision-preview'
+                    },
                     {
                         label: 'gpt-4-vision-preview',
                         name: 'gpt-4-vision-preview'
@@ -220,7 +225,9 @@ class ChatOpenAI_ChatModels implements INode {
 
         const cache = nodeData.inputs?.cache as BaseCache
 
-        const obj: Partial<OpenAIChatInput> & BaseLLMParams & { openAIApiKey?: string } = {
+        const obj: Partial<OpenAIChatInput> &
+            Partial<AzureOpenAIInput> &
+            BaseChatModelParams & { configuration?: ClientOptions & LegacyOpenAIInput; multiModalOption?: IMultiModalOption } = {
             temperature: parseFloat(temperature),
             modelName,
             openAIApiKey,
@@ -244,10 +251,12 @@ class ChatOpenAI_ChatModels implements INode {
             }
         }
 
-        const model = new ChatOpenAI(nodeData.id, obj, {
-            baseURL: basePath,
-            baseOptions: parsedBaseOptions
-        })
+        if (basePath || parsedBaseOptions) {
+            obj.configuration = {
+                baseURL: basePath,
+                baseOptions: parsedBaseOptions
+            }
+        }
 
         const multiModalOption: IMultiModalOption = {
             image: {
@@ -255,7 +264,10 @@ class ChatOpenAI_ChatModels implements INode {
                 imageResolution
             }
         }
-        model.multiModalOption = multiModalOption
+        obj.multiModalOption = multiModalOption
+
+        const model = new ChatOpenAI(nodeData.id, obj)
+
         return model
     }
 }

packages/components/nodes/chatmodels/ChatOpenAI/FlowiseChatOpenAI.ts

@@ -1,10 +1,15 @@
-import { ChatOpenAI as LangchainChatOpenAI, OpenAIChatInput } from 'langchain/chat_models/openai'
-import { BaseChatModelParams } from 'langchain/chat_models/base'
 import type { ClientOptions } from 'openai'
-import type { LegacyOpenAIInput } from '@langchain/openai/dist/types'
-import { BaseLanguageModelInput } from 'langchain/base_language'
-import { ChatOpenAICallOptions } from '@langchain/openai/dist/chat_models'
-import { BaseMessageChunk, BaseMessageLike, HumanMessage, LLMResult } from 'langchain/schema'
+import {
+    ChatOpenAI as LangchainChatOpenAI,
+    OpenAIChatInput,
+    LegacyOpenAIInput,
+    AzureOpenAIInput,
+    ChatOpenAICallOptions
+} from '@langchain/openai'
+import { BaseChatModelParams } from '@langchain/core/language_models/chat_models'
+import { BaseLanguageModelInput } from '@langchain/core/language_models/base'
+import { BaseMessageChunk, BaseMessageLike, HumanMessage } from '@langchain/core/messages'
+import { LLMResult } from '@langchain/core/outputs'
 import { Callbacks } from '@langchain/core/callbacks/manager'
 import { IMultiModalOption } from '../../../src'
 import { addImagesToMessages, MultiModalOptions } from '../../../src/multiModalUtils'
@@ -17,7 +22,9 @@ export class ChatOpenAI extends LangchainChatOpenAI {
 
     constructor(
         id: string,
-        fields?: Partial<OpenAIChatInput> & BaseChatModelParams & { openAIApiKey?: string; multiModalOption?: IMultiModalOption },
+        fields?: Partial<OpenAIChatInput> &
+            Partial<AzureOpenAIInput> &
+            BaseChatModelParams & { configuration?: ClientOptions & LegacyOpenAIInput; multiModalOption?: IMultiModalOption },
         /** @deprecated */
         configuration?: ClientOptions & LegacyOpenAIInput
     ) {

packages/components/nodes/chatmodels/ChatOpenAICustom/ChatOpenAICustom.ts

@@ -1,8 +1,8 @@
+import { ChatOpenAI, OpenAIChatInput } from '@langchain/openai'
+import { BaseCache } from '@langchain/core/caches'
+import { BaseLLMParams } from '@langchain/core/language_models/llms'
 import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
 import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
-import { ChatOpenAI, OpenAIChatInput } from 'langchain/chat_models/openai'
-import { BaseCache } from 'langchain/schema'
-import { BaseLLMParams } from 'langchain/llms/base'
 
 class ChatOpenAICustom_ChatModels implements INode {
     label: string