Feature/Indexing (#1802)

* indexing

* fix for multiple files upsert

* fix default Postgres port

* fix SQLite node description

* add MySQLRecordManager node

* fix MySQL unique index

* add upsert history

* update jsx ui

* lint-fix

* update dialog details

* update llamaindex pinecone

---------

Co-authored-by: chungyau97 <chungyau97@gmail.com>

packages/server/src/services/chatflows/index.ts

@@ -14,6 +14,9 @@ import logger from '../../utils/logger'
 import { getStoragePath } from 'flowise-components'
 import { IReactFlowObject } from '../../Interface'
 import { utilGetUploadsConfig } from '../../utils/getUploadsConfig'
+import { ChatMessage } from '../../database/entities/ChatMessage'
+import { ChatMessageFeedback } from '../../database/entities/ChatMessageFeedback'
+import { UpsertHistory } from '../../database/entities/UpsertHistory'
 
 // Check if chatflow valid for streaming
 const checkIfChatflowIsValidForStreaming = async (chatflowId: string): Promise<any> => {
@@ -105,9 +108,18 @@ const deleteChatflow = async (chatflowId: string): Promise<any> => {
         const appServer = getRunningExpressApp()
         const dbResponse = await appServer.AppDataSource.getRepository(ChatFlow).delete({ id: chatflowId })
         try {
-            // Delete all  uploads corresponding to this chatflow
+            // Delete all uploads corresponding to this chatflow
             const directory = path.join(getStoragePath(), chatflowId)
             deleteFolderRecursive(directory)
+
+            // Delete all chat messages
+            await appServer.AppDataSource.getRepository(ChatMessage).delete({ chatflowid: chatflowId })
+
+            // Delete all chat feedback
+            await appServer.AppDataSource.getRepository(ChatMessageFeedback).delete({ chatflowid: chatflowId })
+
+            // Delete all upsert history
+            await appServer.AppDataSource.getRepository(UpsertHistory).delete({ chatflowid: chatflowId })
         } catch (e) {
             logger.error(`[server]: Error deleting file storage for chatflow ${chatflowId}: ${e}`)
         }

packages/server/src/services/upsert-history/index.ts

@@ -0,0 +1,66 @@
+import { MoreThanOrEqual, LessThanOrEqual } from 'typeorm'
+import { getRunningExpressApp } from '../../utils/getRunningExpressApp'
+import { UpsertHistory } from '../../database/entities/UpsertHistory'
+
+const getAllUpsertHistory = async (
+    sortOrder: string | undefined,
+    chatflowid: string | undefined,
+    startDate: string | undefined,
+    endDate: string | undefined
+) => {
+    try {
+        const appServer = getRunningExpressApp()
+
+        const setDateToStartOrEndOfDay = (dateTimeStr: string, setHours: 'start' | 'end') => {
+            const date = new Date(dateTimeStr)
+            if (isNaN(date.getTime())) {
+                return undefined
+            }
+            setHours === 'start' ? date.setHours(0, 0, 0, 0) : date.setHours(23, 59, 59, 999)
+            return date
+        }
+
+        let fromDate
+        if (startDate) fromDate = setDateToStartOrEndOfDay(startDate, 'start')
+
+        let toDate
+        if (endDate) toDate = setDateToStartOrEndOfDay(endDate, 'end')
+
+        let upsertHistory = await appServer.AppDataSource.getRepository(UpsertHistory).find({
+            where: {
+                chatflowid,
+                ...(fromDate && { date: MoreThanOrEqual(fromDate) }),
+                ...(toDate && { date: LessThanOrEqual(toDate) })
+            },
+            order: {
+                date: sortOrder === 'DESC' ? 'DESC' : 'ASC'
+            }
+        })
+        upsertHistory = upsertHistory.map((hist) => {
+            return {
+                ...hist,
+                result: hist.result ? JSON.parse(hist.result) : {},
+                flowData: hist.flowData ? JSON.parse(hist.flowData) : {}
+            }
+        })
+
+        return upsertHistory
+    } catch (error) {
+        throw new Error(`Error: upsertHistoryServices.getAllUpsertHistory - ${error}`)
+    }
+}
+
+const patchDeleteUpsertHistory = async (ids: string[] = []): Promise<any> => {
+    try {
+        const appServer = getRunningExpressApp()
+        const dbResponse = await appServer.AppDataSource.getRepository(UpsertHistory).delete(ids)
+        return dbResponse
+    } catch (error) {
+        throw new Error(`Error: upsertHistoryServices.patchUpsertHistory - ${error}`)
+    }
+}
+
+export default {
+    getAllUpsertHistory,
+    patchDeleteUpsertHistory
+}

packages/server/src/services/variables/index.ts

@@ -18,7 +18,7 @@ const deleteVariable = async (variableId: string): Promise<any> => {
         const dbResponse = await appServer.AppDataSource.getRepository(Variable).delete({ id: variableId })
         return dbResponse
     } catch (error) {
-        throw new Error(`Error: variablesServices.createVariable - ${error}`)
+        throw new Error(`Error: variablesServices.deleteVariable - ${error}`)
     }
 }