Feature/Add bullmq redis for message queue processing (#3568)

* add bullmq redis for message queue processing

* Update pnpm-lock.yaml

* update queue manager

* remove singleton patterns, add redis to cache pool

* add bull board ui

* update rate limit handler

* update redis configuration

* Merge add rate limit redis prefix

* update rate limit queue events

* update preview loader to queue

* refractor namings to constants

* update env variable for queue

* update worker shutdown gracefully

packages/server/package.json

@@ -26,6 +26,8 @@
         "nuke": "rimraf dist node_modules .turbo",
         "start:windows": "cd bin && run start",
         "start:default": "cd bin && ./run start",
+        "start-worker:windows": "cd bin && run worker",
+        "start-worker:default": "cd bin && ./run worker",
         "dev": "tsc-watch --noClear -p ./tsconfig.json --onSuccess \"pnpm start\"",
         "oclif-dev": "run-script-os",
         "oclif-dev:windows": "cd bin && dev start",
@@ -55,7 +57,7 @@
     "license": "SEE LICENSE IN LICENSE.md",
     "dependencies": {
         "@aws-sdk/client-secrets-manager": "^3.699.0",
-        "@oclif/core": "^1.13.10",
+        "@oclif/core": "4.0.7",
         "@opentelemetry/api": "^1.3.0",
         "@opentelemetry/auto-instrumentations-node": "^0.52.0",
         "@opentelemetry/core": "1.27.0",
@@ -74,6 +76,8 @@
         "@types/uuid": "^9.0.7",
         "async-mutex": "^0.4.0",
         "axios": "1.6.2",
+        "bull-board": "^2.1.3",
+        "bullmq": "^5.13.2",
         "content-disposition": "0.5.4",
         "cors": "^2.8.5",
         "crypto-js": "^4.1.1",
@@ -97,10 +101,10 @@
         "pg": "^8.11.1",
         "posthog-node": "^3.5.0",
         "prom-client": "^15.1.3",
+        "rate-limit-redis": "^4.2.0",
         "reflect-metadata": "^0.1.13",
         "s3-streamlogger": "^1.11.0",
         "sanitize-html": "^2.11.0",
-        "socket.io": "^4.6.1",
         "sqlite3": "^5.1.6",
         "typeorm": "^0.3.6",
         "uuid": "^9.0.1",