NVIDIA NIM fixes (#4215)

* fix: udpate label to "NVIDIA NIM API Key"

* test: update tag from ":latest" to ":1.8.0-rtx"

* test: add image URL path "nvcr.io/nim/"

* fix/nvidia-nim-2 (#4208)

* fix: update nim-container-manager

* feat: add "DeepSeek R1 Distill Llama 8B"

* fix/nidia-nim-3 (#4209)

* chore: add error message NVIDIA NIM is not installed.

* chore: standardize NVIDIA NGC API Key

* chore: capitalize Nvidia to NVIDIA

* chore: generalize error message for chat models

* fix/nvidia-nim-4-yau (#4212)

* test: nimRelaxMemConstraints and hostPort

* test: add logger for hostPort and nimRelaxMemConstraints

* test: nim-container-manager version 1.0.9

* test: parseInt nimRelaxMemConstraints

* test: update nim-container-manager version to 1.0.10

* chore: update nim-container-manager version to 1.0.11

* Update start container behaviour - show existing containers and give users the choice

* Go back to previous step when clicking start new so user can change port number

* Update condition for showing existing container dialog

* Fix start new in different port not working

* Update get container controller

* Update again

* fix: generalize error message for chat models

* Update getContainer controller

* Fix incorrect image check in getContainer controller

* Update existing container dialog text

* Fix styles in container exists dialog for nvidia nim

---------

Co-authored-by: chungyau97 <chungyau97@gmail.com>
Co-authored-by: Ong Chung Yau <33013947+chungyau97@users.noreply.github.com>

packages/server/package.json

@@ -97,7 +97,7 @@
         "multer": "^1.4.5-lts.1",
         "multer-s3": "^3.0.1",
         "mysql2": "^3.11.3",
-        "nim-container-manager": "^1.0.5",
+        "flowise-nim-container-manager": "^1.0.11",
         "openai": "^4.82.0",
         "pg": "^8.11.1",
         "posthog-node": "^3.5.0",

packages/server/src/controllers/nvidia-nim/index.ts

@@ -1,7 +1,7 @@
 import axios from 'axios'
-import { Request, Response, NextFunction } from 'express'
+import { NextFunction, Request, Response } from 'express'
 
-const { NimContainerManager } = require('nim-container-manager')
+const { NimContainerManager } = require('flowise-nim-container-manager')
 
 const getToken = async (req: Request, res: Response, next: NextFunction) => {
     try {
@@ -55,7 +55,13 @@ const startContainer = async (req: Request, res: Response, next: NextFunction) =
     try {
         const imageTag = req.body.imageTag
         const apiKey = req.body.apiKey
-        await NimContainerManager.startContainer(imageTag, apiKey)
+        const hostPort = req.body.hostPort
+        const nimRelaxMemConstraints = parseInt(req.body.nimRelaxMemConstraints)
+        // Validate nimRelaxMemConstraints
+        if (isNaN(nimRelaxMemConstraints) || (nimRelaxMemConstraints !== 0 && nimRelaxMemConstraints !== 1)) {
+            return res.status(400).send('nimRelaxMemConstraints must be 0 or 1')
+        }
+        await NimContainerManager.startContainer(imageTag, apiKey, hostPort, nimRelaxMemConstraints)
         return res.send(`Starting container ${imageTag}`)
     } catch (error) {
         next(error)
@@ -79,17 +85,51 @@ const getImage = async (req: Request, res: Response, next: NextFunction) => {
 const getContainer = async (req: Request, res: Response, next: NextFunction) => {
     try {
         const imageTag = req.body.imageTag
+        const port = req.body.port
+
+        // First check if the image exists
         const images = await NimContainerManager.userImageLibrary()
         const image = images.find((img: any) => img.tag === imageTag)
         if (!image) {
             return res.status(404).send(`Image ${imageTag} not found`)
         }
-        if (!image.container) {
-            return res.status(404).send(`Container of ${imageTag} not found`)
+
+        const containers = await NimContainerManager.listRunningContainers()
+        const portInUse = containers.find((cont: any) => cont.port === port)
+        if (portInUse) {
+            const isModelContainer = portInUse.image === image.tag
+            if (isModelContainer) {
+                portInUse.image = image.name
+                return res.json(portInUse)
+            } else {
+                return res.status(409).send({
+                    message: `Port ${port} is already in use by another container`,
+                    container: portInUse
+                })
+            }
         }
-        const container = image.container
-        container.image = image.name
-        return res.json(container)
+
+        // If no container found with matching port, return 404
+        return res.status(404).send(`Container of ${imageTag} with port ${port} not found`)
+    } catch (error) {
+        next(error)
+    }
+}
+
+const listRunningContainers = async (req: Request, res: Response, next: NextFunction) => {
+    try {
+        const containers = await NimContainerManager.listRunningContainers()
+        return res.json(containers)
+    } catch (error) {
+        next(error)
+    }
+}
+
+const stopContainer = async (req: Request, res: Response, next: NextFunction) => {
+    try {
+        const containerId = req.body.containerId
+        const containerInfo = await NimContainerManager.stopContainer(containerId)
+        return res.json(containerInfo)
     } catch (error) {
         next(error)
     }
@@ -102,5 +142,7 @@ export default {
     pullImage,
     startContainer,
     getImage,
-    getContainer
+    getContainer,
+    listRunningContainers,
+    stopContainer
 }

packages/server/src/middlewares/errors/index.ts

@@ -5,6 +5,8 @@ import { InternalFlowiseError } from '../../errors/internalFlowiseError'
 // we need eslint because we have to pass next arg for the error middleware
 // eslint-disable-next-line
 async function errorHandlerMiddleware(err: InternalFlowiseError, req: Request, res: Response, next: NextFunction) {
+    if (err.message.includes('401 Incorrect API key provided'))
+        err.message = '401 Invalid model key or Incorrect local model configuration.'
     let displayedError = {
         statusCode: err.statusCode || StatusCodes.INTERNAL_SERVER_ERROR,
         success: false,

packages/server/src/routes/nvidia-nim/index.ts

@@ -6,8 +6,10 @@ const router = express.Router()
 router.get('/preload', nimController.preload)
 router.get('/get-token', nimController.getToken)
 router.get('/download-installer', nimController.downloadInstaller)
+router.get('/list-running-containers', nimController.listRunningContainers)
 router.post('/pull-image', nimController.pullImage)
 router.post('/start-container', nimController.startContainer)
+router.post('/stop-container', nimController.stopContainer)
 router.post('/get-image', nimController.getImage)
 router.post('/get-container', nimController.getContainer)
 

packages/server/src/utils/SSEStreamer.ts

@@ -166,6 +166,7 @@ export class SSEStreamer implements IServerSideEventStreamer {
     }
 
     streamErrorEvent(chatId: string, msg: string) {
+        if (msg.includes('401 Incorrect API key provided')) msg = '401 Invalid model key or Incorrect local model configuration.'
         const client = this.clients[chatId]
         if (client) {
             const clientResponse = {