farcasclaudiu/Flowise
1
0
Fork 0
mirror of https://github.com/farcasclaudiu/Flowise.git synced 2026-06-28 15:00:57 +03:00
Code Issues Packages Projects Releases Wiki Activity

Feature/Allow winston to stream to s3 (#3848)

Browse Source 
allow winston to stream to s3
This commit is contained in:
Henry Heng
2025-01-11 17:01:10 +00:00
committed by GitHub
parent a2d5cf979c
commit 15d06ec4b3
3 changed files with 131 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/server/package.json
+1
View File
@@ -97,6 +97,7 @@
"posthog-node": "^3.5.0",
"prom-client": "^15.1.3",
"reflect-metadata": "^0.1.13",
"s3-streamlogger": "^1.11.0",
"sanitize-html": "^2.11.0",
"socket.io": "^4.6.1",
"sqlite3": "^5.1.6",
packages/server/src/utils/logger.ts
+102 -24
View File
@@ -1,11 +1,51 @@
import * as path from 'path'
import * as fs from 'fs'
import { hostname } from 'node:os'
import config from './config' // should be replaced by node-config or similar
import { createLogger, transports, format } from 'winston'
import { NextFunction, Request, Response } from 'express'
const { S3StreamLogger } = require('s3-streamlogger')
const { combine, timestamp, printf, errors } = format
let s3ServerStream: any
let s3ErrorStream: any
let s3ServerReqStream: any
if (process.env.STORAGE_TYPE === 's3') {
const accessKeyId = process.env.S3_STORAGE_ACCESS_KEY_ID
const secretAccessKey = process.env.S3_STORAGE_SECRET_ACCESS_KEY
const region = process.env.S3_STORAGE_REGION
const s3Bucket = process.env.S3_STORAGE_BUCKET_NAME
s3ServerStream = new S3StreamLogger({
bucket: s3Bucket,
folder: 'logs/server',
region,
access_key_id: accessKeyId,
secret_access_key: secretAccessKey,
name_format: `server-%Y-%m-%d-%H-%M-%S-%L-${hostname()}.log`
})
s3ErrorStream = new S3StreamLogger({
bucket: s3Bucket,
folder: 'logs/error',
region,
access_key_id: accessKeyId,
secret_access_key: secretAccessKey,
name_format: `server-error-%Y-%m-%d-%H-%M-%S-%L-${hostname()}.log`
})
s3ServerReqStream = new S3StreamLogger({
bucket: s3Bucket,
folder: 'logs/requests',
region,
access_key_id: accessKeyId,
secret_access_key: secretAccessKey,
name_format: `server-requests-%Y-%m-%d-%H-%M-%S-%L-${hostname()}.log.jsonl`
})
}
// expect the log dir be relative to the projects root
const logDir = config.logging.dir
@@ -29,33 +69,60 @@ const logger = createLogger({
},
transports: [
new transports.Console(),
new transports.File({
filename: path.join(logDir, config.logging.server.filename ?? 'server.log'),
level: config.logging.server.level ?? 'info'
}),
new transports.File({
filename: path.join(logDir, config.logging.server.errorFilename ?? 'server-error.log'),
level: 'error' // Log only errors to this file
})
...(!process.env.STORAGE_TYPE || process.env.STORAGE_TYPE === 'local'
? [
new transports.File({
filename: path.join(logDir, config.logging.server.filename ?? 'server.log'),
level: config.logging.server.level ?? 'info'
}),
new transports.File({
filename: path.join(logDir, config.logging.server.errorFilename ?? 'server-error.log'),
level: 'error' // Log only errors to this file
})
]
: []),
...(process.env.STORAGE_TYPE === 's3'
? [
new transports.Stream({
stream: s3ServerStream
})
]
: [])
],
exceptionHandlers: [
new transports.File({
filename: path.join(logDir, config.logging.server.errorFilename ?? 'server-error.log')
})
...(!process.env.STORAGE_TYPE || process.env.STORAGE_TYPE === 'local'
? [
new transports.File({
filename: path.join(logDir, config.logging.server.errorFilename ?? 'server-error.log')
})
]
: []),
...(process.env.STORAGE_TYPE === 's3'
? [
new transports.Stream({
stream: s3ErrorStream
})
]
: [])
],
rejectionHandlers: [
new transports.File({
filename: path.join(logDir, config.logging.server.errorFilename ?? 'server-error.log')
})
...(!process.env.STORAGE_TYPE || process.env.STORAGE_TYPE === 'local'
? [
new transports.File({
filename: path.join(logDir, config.logging.server.errorFilename ?? 'server-error.log')
})
]
: []),
...(process.env.STORAGE_TYPE === 's3'
? [
new transports.Stream({
stream: s3ErrorStream
})
]
: [])
]
})
/**
* This function is used by express as a middleware.
* @example
* this.app = express()
* this.app.use(expressRequestLogger)
*/
export function expressRequestLogger(req: Request, res: Response, next: NextFunction): void {
const unwantedLogURLs = ['/api/v1/node-icon/', '/api/v1/components-credentials-icon/']
if (/\/api\/v1\//i.test(req.url) && !unwantedLogURLs.some((url) => new RegExp(url, 'i').test(req.url))) {
@@ -73,10 +140,21 @@ export function expressRequestLogger(req: Request, res: Response, next: NextFunc
}
},
transports: [
new transports.File({
filename: path.join(logDir, config.logging.express.filename ?? 'server-requests.log.jsonl'),
level: config.logging.express.level ?? 'debug'
})
...(!process.env.STORAGE_TYPE || process.env.STORAGE_TYPE === 'local'
? [
new transports.File({
filename: path.join(logDir, config.logging.express.filename ?? 'server-requests.log.jsonl'),
level: config.logging.express.level ?? 'debug'
})
]
: []),
...(process.env.STORAGE_TYPE === 's3'
? [
new transports.Stream({
stream: s3ServerReqStream
})
]
: [])
]
})