GPT Vision: Initial implementation of the OpenAI Vision API

packages/components/nodes/chains/VisionChain/OpenAIVisionChain.ts

@@ -12,6 +12,7 @@ class OpenAIVisionChain_Chains implements INode {
     version: number
     type: string
     icon: string
+    badge: string
     category: string
     baseClasses: string[]
     description: string
@@ -21,10 +22,11 @@ class OpenAIVisionChain_Chains implements INode {
     constructor() {
         this.label = 'Open AI Vision Chain'
         this.name = 'openAIVisionChain'
-        this.version = 3.0
+        this.version = 1.0
         this.type = 'OpenAIVisionChain'
         this.icon = 'chain.svg'
         this.category = 'Chains'
+        this.badge = 'EXPERIMENTAL'
         this.description = 'Chain to run queries against OpenAI (GPT-4) Vision .'
         this.baseClasses = [this.type, ...getBaseClasses(VLLMChain)]
         this.inputs = [
@@ -63,6 +65,20 @@ class OpenAIVisionChain_Chains implements INode {
                 type: 'string',
                 placeholder: 'Name Your Chain',
                 optional: true
+            },
+            {
+                label: 'Accepted Upload Types',
+                name: 'allowedUploadTypes',
+                type: 'string',
+                default: 'image/gif;image/jpeg;image/png;image/webp',
+                hidden: true
+            },
+            {
+                label: 'Maximum Upload Size (MB)',
+                name: 'maxUploadSize',
+                type: 'number',
+                default: '5',
+                hidden: true
             }
         ]
         this.outputs = [
@@ -93,7 +109,7 @@ class OpenAIVisionChain_Chains implements INode {
             openAIApiKey: openAIModel.openAIApiKey,
             imageResolution: imageResolution,
             verbose: process.env.DEBUG === 'true',
-            imageUrls: options.url,
+            imageUrls: options.uploads,
             openAIModel: openAIModel
         }
         if (output === this.name) {
@@ -156,8 +172,8 @@ const runPrediction = async (
      * TO: { "value": "hello i am ben\n\n\thow are you?" }
      */
     const promptValues = handleEscapeCharacters(promptValuesRaw, true)
-    if (options?.url) {
-        chain.imageUrls = options.url
+    if (options?.uploads) {
+        chain.imageUrls = options.uploads
     }
     if (promptValues && inputVariables.length > 0) {
         let seen: string[] = []

packages/components/nodes/chains/VisionChain/VLLMChain.ts

@@ -79,6 +79,7 @@ export class VLLMChain extends BaseChain implements OpenAIVisionChainInput {
             messages: []
         }
         if (this.openAIModel.maxTokens) vRequest.max_tokens = this.openAIModel.maxTokens
+        else vRequest.max_tokens = 1024
 
         const userRole: any = { role: 'user' }
         userRole.content = []

packages/components/nodes/chains/VisionChain/chain.svg

@@ -0,0 +1,6 @@
+<svg xmlns="http://www.w3.org/2000/svg" class="icon icon-tabler icon-tabler-dna" width="24" height="24" viewBox="0 0 24 24" stroke-width="2" stroke="currentColor" fill="none" stroke-linecap="round" stroke-linejoin="round">
+   <path stroke="none" d="M0 0h24v24H0z" fill="none"></path>
+   <path d="M14.828 14.828a4 4 0 1 0 -5.656 -5.656a4 4 0 0 0 5.656 5.656z"></path>
+   <path d="M9.172 20.485a4 4 0 1 0 -5.657 -5.657"></path>
+   <path d="M14.828 3.515a4 4 0 0 0 5.657 5.657"></path>
+</svg>