Document Store - Phase 2 (#2912)

* Document Store - Phase 2

* Adding additional columns for vector store config, document store phase 2

* Adding additional columns for vector store config, document store phase 2

* Document Store - Phase 2 - Upsert and Query

* ux cleanup

* retrieval settings and more ux changes

* adding MMR params to execution

* Making the upsert process async.

* add upsert history changes

* making the searchParams dynamic

* removing unnecessary params

* add ability to delete data from vector store

* update margin for vector store query

* adding option to save config in the retrieval playground

* adding chunk number for query return chunks

* Adding a Document Store node in the VectorStore category

* update doc store status, ui touchup

---------

Co-authored-by: Henry <hzj94@hotmail.com>

packages/server/src/Interface.DocumentStore.ts

@@ -5,7 +5,9 @@ export enum DocumentStoreStatus {
     SYNC = 'SYNC',
     SYNCING = 'SYNCING',
     STALE = 'STALE',
-    NEW = 'NEW'
+    NEW = 'NEW',
+    UPSERTING = 'UPSERTING',
+    UPSERTED = 'UPSERTED'
 }
 
 export interface IDocumentStore {
@@ -17,6 +19,9 @@ export interface IDocumentStore {
     updatedDate: Date
     createdDate: Date
     status: DocumentStoreStatus
+    vectorStoreConfig: string | null // JSON string
+    embeddingConfig: string | null // JSON string
+    recordManagerConfig: string | null // JSON string
 }
 
 export interface IDocumentStoreFileChunk {
@@ -89,6 +94,9 @@ export class DocumentStoreDTO {
     totalChars: number
     chunkSize: number
     loaders: IDocumentStoreLoader[]
+    vectorStoreConfig: any
+    embeddingConfig: any
+    recordManagerConfig: any
 
     constructor() {}
 
@@ -109,6 +117,16 @@ export class DocumentStoreDTO {
             documentStoreDTO.whereUsed = []
         }
 
+        if (entity.vectorStoreConfig) {
+            documentStoreDTO.vectorStoreConfig = JSON.parse(entity.vectorStoreConfig)
+        }
+        if (entity.embeddingConfig) {
+            documentStoreDTO.embeddingConfig = JSON.parse(entity.embeddingConfig)
+        }
+        if (entity.recordManagerConfig) {
+            documentStoreDTO.recordManagerConfig = JSON.parse(entity.recordManagerConfig)
+        }
+
         if (entity.loaders) {
             documentStoreDTO.loaders = JSON.parse(entity.loaders)
             documentStoreDTO.loaders.map((loader) => {