farcasclaudiu/Flowise
1
0
Fork 0
mirror of https://github.com/farcasclaudiu/Flowise.git synced 2026-06-28 17:01:00 +03:00
Code Issues Packages Projects Releases Wiki Activity

Document Store - Phase 2 (#2912)

Browse Source 
* Document Store - Phase 2

* Adding additional columns for vector store config, document store phase 2

* Adding additional columns for vector store config, document store phase 2

* Document Store - Phase 2 - Upsert and Query

* ux cleanup

* retrieval settings and more ux changes

* adding MMR params to execution

* Making the upsert process async.

* add upsert history changes

* making the searchParams dynamic

* removing unnecessary params

* add ability to delete data from vector store

* update margin for vector store query

* adding option to save config in the retrieval playground

* adding chunk number for query return chunks

* Adding a Document Store node in the VectorStore category

* update doc store status, ui touchup

---------

Co-authored-by: Henry <hzj94@hotmail.com>
This commit is contained in:
Vinod Kiran
2024-08-07 23:29:52 +05:30
committed by GitHub
parent c7306c93d7
commit c0bae635b0
36 changed files with 3589 additions and 91 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/server/src/database/entities/DocumentStore.ts
+9
View File
@@ -28,4 +28,13 @@ export class DocumentStore implements IDocumentStore {
@Column({ nullable: false, type: 'text' })
status: DocumentStoreStatus
@Column({ nullable: true, type: 'text' })
vectorStoreConfig: string | null
@Column({ nullable: true, type: 'text' })
embeddingConfig: string | null
@Column({ nullable: true, type: 'text' })
recordManagerConfig: string | null
}
packages/server/src/database/migrations/mysql/1715861032479-AddVectorStoreConfigToDocStore.ts
+18
View File
@@ -0,0 +1,18 @@
import { MigrationInterface, QueryRunner } from 'typeorm'
export class AddVectorStoreConfigToDocStore1715861032479 implements MigrationInterface {
public async up(queryRunner: QueryRunner): Promise<void> {
const columnExists = await queryRunner.hasColumn('document_store', 'vectorStoreConfig')
if (!columnExists) {
await queryRunner.query(`ALTER TABLE \`document_store\` ADD COLUMN \`vectorStoreConfig\` TEXT;`)
await queryRunner.query(`ALTER TABLE \`document_store\` ADD COLUMN \`embeddingConfig\` TEXT;`)
await queryRunner.query(`ALTER TABLE \`document_store\` ADD COLUMN \`recordManagerConfig\` TEXT;`)
}
}
public async down(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`ALTER TABLE \`document_store\` DROP COLUMN \`vectorStoreConfig\`;`)
await queryRunner.query(`ALTER TABLE \`document_store\` DROP COLUMN \`embeddingConfig\`;`)
await queryRunner.query(`ALTER TABLE \`document_store\` DROP COLUMN \`recordManagerConfig\`;`)
}
}
packages/server/src/database/migrations/mysql/index.ts
+2
View File
@@ -17,6 +17,7 @@ import { AddFeedback1707213626553 } from './1707213626553-AddFeedback'
import { AddUpsertHistoryEntity1709814301358 } from './1709814301358-AddUpsertHistoryEntity'
import { AddLead1710832127079 } from './1710832127079-AddLead'
import { AddLeadToChatMessage1711538023578 } from './1711538023578-AddLeadToChatMessage'
import { AddVectorStoreConfigToDocStore1715861032479 } from './1715861032479-AddVectorStoreConfigToDocStore'
import { AddDocumentStore1711637331047 } from './1711637331047-AddDocumentStore'
import { AddAgentReasoningToChatMessage1714679514451 } from './1714679514451-AddAgentReasoningToChatMessage'
import { AddTypeToChatFlow1716300000000 } from './1716300000000-AddTypeToChatFlow'
@@ -47,6 +48,7 @@ export const mysqlMigrations = [
AddLeadToChatMessage1711538023578,
AddAgentReasoningToChatMessage1714679514451,
AddTypeToChatFlow1716300000000,
AddVectorStoreConfigToDocStore1715861032479,
AddApiKey1720230151480,
AddActionToChatMessage1721078251523,
LongTextColumn1722301395521
packages/server/src/database/migrations/postgres/1715861032479-AddVectorStoreConfigToDocStore.ts
+15
View File
@@ -0,0 +1,15 @@
import { MigrationInterface, QueryRunner } from 'typeorm'
export class AddVectorStoreConfigToDocStore1715861032479 implements MigrationInterface {
public async up(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`ALTER TABLE "document_store" ADD COLUMN IF NOT EXISTS "vectorStoreConfig" TEXT;`)
await queryRunner.query(`ALTER TABLE "document_store" ADD COLUMN IF NOT EXISTS "embeddingConfig" TEXT;`)
await queryRunner.query(`ALTER TABLE "document_store" ADD COLUMN IF NOT EXISTS "recordManagerConfig" TEXT;`)
}
public async down(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`ALTER TABLE "document_store" DROP COLUMN "vectorStoreConfig";`)
await queryRunner.query(`ALTER TABLE "document_store" DROP COLUMN "embeddingConfig";`)
await queryRunner.query(`ALTER TABLE "document_store" DROP COLUMN "recordManagerConfig";`)
}
}
packages/server/src/database/migrations/postgres/index.ts
+2
View File
@@ -18,6 +18,7 @@ import { AddUpsertHistoryEntity1709814301358 } from './1709814301358-AddUpsertHi
import { FieldTypes1710497452584 } from './1710497452584-FieldTypes'
import { AddLead1710832137905 } from './1710832137905-AddLead'
import { AddLeadToChatMessage1711538016098 } from './1711538016098-AddLeadToChatMessage'
import { AddVectorStoreConfigToDocStore1715861032479 } from './1715861032479-AddVectorStoreConfigToDocStore'
import { AddDocumentStore1711637331047 } from './1711637331047-AddDocumentStore'
import { AddAgentReasoningToChatMessage1714679514451 } from './1714679514451-AddAgentReasoningToChatMessage'
import { AddTypeToChatFlow1716300000000 } from './1716300000000-AddTypeToChatFlow'
@@ -48,6 +49,7 @@ export const postgresMigrations = [
AddLeadToChatMessage1711538016098,
AddAgentReasoningToChatMessage1714679514451,
AddTypeToChatFlow1716300000000,
AddVectorStoreConfigToDocStore1715861032479,
AddApiKey1720230151480,
AddActionToChatMessage1721078251523
]
packages/server/src/database/migrations/sqlite/1715861032479-AddVectorStoreConfigToDocStore.ts
+15
View File
@@ -0,0 +1,15 @@
import { MigrationInterface, QueryRunner } from 'typeorm'
export class AddVectorStoreConfigToDocStore1715861032479 implements MigrationInterface {
public async up(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`ALTER TABLE "document_store" ADD COLUMN "vectorStoreConfig" TEXT;`)
await queryRunner.query(`ALTER TABLE "document_store" ADD COLUMN "embeddingConfig" TEXT;`)
await queryRunner.query(`ALTER TABLE "document_store" ADD COLUMN "recordManagerConfig" TEXT;`)
}
public async down(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`ALTER TABLE "document_store" DROP COLUMN "vectorStoreConfig";`)
await queryRunner.query(`ALTER TABLE "document_store" DROP COLUMN "embeddingConfig";`)
await queryRunner.query(`ALTER TABLE "document_store" DROP COLUMN "recordManagerConfig";`)
}
}
packages/server/src/database/migrations/sqlite/index.ts
+2
View File
@@ -17,6 +17,7 @@ import { AddFeedback1707213619308 } from './1707213619308-AddFeedback'
import { AddUpsertHistoryEntity1709814301358 } from './1709814301358-AddUpsertHistoryEntity'
import { AddLead1710832117612 } from './1710832117612-AddLead'
import { AddLeadToChatMessage1711537986113 } from './1711537986113-AddLeadToChatMessage'
import { AddVectorStoreConfigToDocStore1715861032479 } from './1715861032479-AddVectorStoreConfigToDocStore'
import { AddDocumentStore1711637331047 } from './1711637331047-AddDocumentStore'
import { AddAgentReasoningToChatMessage1714679514451 } from './1714679514451-AddAgentReasoningToChatMessage'
import { AddTypeToChatFlow1716300000000 } from './1716300000000-AddTypeToChatFlow'
@@ -46,6 +47,7 @@ export const sqliteMigrations = [
AddLeadToChatMessage1711537986113,
AddAgentReasoningToChatMessage1714679514451,
AddTypeToChatFlow1716300000000,
AddVectorStoreConfigToDocStore1715861032479,
AddApiKey1720230151480,
AddActionToChatMessage1721078251523
]