Added winston logging

- use logger.xxx instead of console.xxx
- added express middleware logging (using jsonl)
- added LOG_PATH as environment variable
- more configs postponed for later iteration

packages/server/src/utils/config.ts

@@ -0,0 +1,25 @@
+// BEWARE: This file is an intereem solution until we have a proper config strategy
+
+import path from 'path'
+import dotenv from 'dotenv'
+
+dotenv.config({ path: path.join(__dirname, '..', '..', '.env'), override: true })
+
+// default config
+const loggingConfig = {
+    dir: process.env.LOG_PATH ?? './logs',
+    server: {
+        level: 'info',
+        filename: 'server.log',
+        errorFilename: 'server-error.log'
+    },
+    express: {
+        level: 'info',
+        format: 'jsonl', // can't be changed currently
+        filename: 'server-requests.log.jsonl' // should end with .jsonl
+    }
+}
+
+export default {
+    logging: loggingConfig
+}

packages/server/src/utils/index.ts

@@ -1,6 +1,7 @@
 import path from 'path'
 import fs from 'fs'
 import moment from 'moment'
+import logger from './logger'
 import {
     IComponentNodes,
     IDepthQueue,
@@ -227,7 +228,7 @@ export const buildLangchain = async (
                 databaseEntities
             })
         } catch (e: any) {
-            console.error(e)
+            logger.error(e)
             throw new Error(e)
         }
 
@@ -595,7 +596,7 @@ export const replaceAllAPIKeys = async (content: ICommonObject[]): Promise<void>
     try {
         await fs.promises.writeFile(getAPIKeyPath(), JSON.stringify(content), 'utf8')
     } catch (error) {
-        console.error(error)
+        logger.error(error)
     }
 }
 

packages/server/src/utils/logger.ts

@@ -0,0 +1,100 @@
+import * as path from 'path'
+import * as fs from 'fs'
+import config from './config' // should be replaced by node-config or similar
+import { createLogger, transports, format } from 'winston'
+import { NextFunction, Request, Response } from 'express'
+
+const { combine, timestamp, printf } = format
+
+// expect the log dir be relative to the projects root
+const logDir = path.join(__dirname, '../../../..', config.logging.dir ?? './logs')
+
+// Create the log directory if it doesn't exist
+if (!fs.existsSync(logDir)) {
+    fs.mkdirSync(logDir)
+}
+
+const logger = createLogger({
+    format: combine(
+        timestamp({ format: 'YYYY-MM-DD HH:mm:ss' }),
+        format.json(),
+        printf(({ level, message, timestamp }) => {
+            return `${timestamp} [${level.toUpperCase()}]: ${message}`
+        })
+    ),
+    defaultMeta: {
+        package: 'server'
+    },
+    transports: [
+        new transports.Console(),
+        new transports.File({
+            filename: path.join(logDir, config.logging.server.filename ?? 'server.log'),
+            level: config.logging.server.level ?? 'info'
+        }),
+        new transports.File({
+            filename: path.join(logDir, config.logging.server.errorFilename ?? 'server-error.log'),
+            level: 'error' // Log only errors to this file
+        })
+    ],
+    exceptionHandlers: [
+        new transports.File({
+            filename: path.join(logDir, config.logging.server.errorFilename ?? 'server-error.log')
+        })
+    ],
+    rejectionHandlers: [
+        new transports.File({
+            filename: path.join(logDir, config.logging.server.errorFilename ?? 'server-error.log')
+        })
+    ]
+})
+
+/**
+ * This function is used by express as a middleware.
+ * @example
+ *   this.app = express()
+ *   this.app.use(expressRequestLogger)
+ */
+export function expressRequestLogger(req: Request, res: Response, next: NextFunction): void {
+    const fileLogger = createLogger({
+        format: combine(timestamp({ format: 'YYYY-MM-DD HH:mm:ss' }), format.json()),
+        defaultMeta: {
+            package: 'server',
+            request: {
+                method: req.method,
+                url: req.url,
+                body: req.body,
+                query: req.query,
+                params: req.params,
+                headers: req.headers
+            }
+        },
+        transports: [
+            new transports.File({
+                filename: path.join(logDir, config.logging.express.filename ?? 'server-requests.log.jsonl'),
+                level: 'debug'
+            })
+        ]
+    })
+
+    const getRequestEmoji = (method: string) => {
+        const requetsEmojis: Record<string, string> = {
+            GET: '⬇️',
+            POST: '⬆️',
+            PUT: '🖊',
+            DELETE: '❌'
+        }
+
+        return requetsEmojis[method] || '?'
+    }
+
+    if (req.method !== 'GET') {
+        fileLogger.info(`${getRequestEmoji(req.method)} ${req.method} ${req.url}`)
+        logger.info(`${getRequestEmoji(req.method)} ${req.method} ${req.url}`)
+    } else {
+        fileLogger.http(`${getRequestEmoji(req.method)} ${req.method} ${req.url}`)
+    }
+
+    next()
+}
+
+export default logger