Feature/DocumentStore (#2106)

* datasource: initial commit * datasource: datasource details and chunks * datasource: Document Store Node * more changes * Document Store - Base functionality * Document Store Loader Component * Document Store Loader Component * before merging the modularity PR * after merging the modularity PR * preview mode * initial draft PR * fixes * minor updates and fixes * preview with loader and splitter * preview with credential * show stored chunks * preview update... * edit config * save, preview and other changes * save, preview and other changes * save, process and other changes * save, process and other changes * alpha1 - for internal testing * rerouting urls * bug fix on new leader create * pagination support for chunks * delete document store * Update pnpm-lock.yaml * doc store card view * Update store files to use updated storage functions, Document Store Table View and other changes * ui changes * add expanded chunk dialog, improve ui * change throw Error to InternalError * Bug Fixes and removal of subFolder, adding of view chunks for store * lint fixes * merge changes * DocumentStoreStatus component * ui changes for doc store * add remove metadata key field, add custom document loader * add chatflows used doc store chips * add types/interfaces to DocumentStore Services * document loader list dialog title bar color change * update interfaces * Whereused Chatflow Name and Added chunkNo to retain order of created chunks. * use typeorm order chunkNo, ui changes --------- Co-authored-by: Henry <hzj94@hotmail.com> Co-authored-by: Henry Heng <henryheng@flowiseai.com>
2026-06-28 19:00:59 +03:00 · 2024-05-06 19:53:27 +05:30
parent af4e28aa91
commit 40e36d1b39
91 changed files with 38713 additions and 32791 deletions
@@ -27,6 +27,7 @@ class CharacterTextSplitter_TextSplitters implements INode {
                label: 'Chunk Size',
                name: 'chunkSize',
                type: 'number',
+                description: 'Number of characters in each chunk. Default is 1000.',
                default: 1000,
                optional: true
            },
@@ -34,6 +35,8 @@ class CharacterTextSplitter_TextSplitters implements INode {
                label: 'Chunk Overlap',
                name: 'chunkOverlap',
                type: 'number',
+                description: 'Number of characters to overlap between chunks. Default is 200.',
+                default: 200,
                optional: true
            },
            {
@@ -101,6 +101,7 @@ class CodeTextSplitter_TextSplitters implements INode {
                label: 'Chunk Size',
                name: 'chunkSize',
                type: 'number',
+                description: 'Number of characters in each chunk. Default is 1000.',
                default: 1000,
                optional: true
            },
@@ -108,6 +109,8 @@ class CodeTextSplitter_TextSplitters implements INode {
                label: 'Chunk Overlap',
                name: 'chunkOverlap',
                type: 'number',
+                description: 'Number of characters to overlap between chunks. Default is 200.',
+                default: 200,
                optional: true
            }
        ]
@@ -28,6 +28,7 @@ class HtmlToMarkdownTextSplitter_TextSplitters implements INode {
                label: 'Chunk Size',
                name: 'chunkSize',
                type: 'number',
+                description: 'Number of characters in each chunk. Default is 1000.',
                default: 1000,
                optional: true
            },
@@ -35,6 +36,8 @@ class HtmlToMarkdownTextSplitter_TextSplitters implements INode {
                label: 'Chunk Overlap',
                name: 'chunkOverlap',
                type: 'number',
+                description: 'Number of characters to overlap between chunks. Default is 200.',
+                default: 200,
                optional: true
            }
        ]
@@ -27,6 +27,7 @@ class MarkdownTextSplitter_TextSplitters implements INode {
                label: 'Chunk Size',
                name: 'chunkSize',
                type: 'number',
+                description: 'Number of characters in each chunk. Default is 1000.',
                default: 1000,
                optional: true
            },
@@ -34,6 +35,8 @@ class MarkdownTextSplitter_TextSplitters implements INode {
                label: 'Chunk Overlap',
                name: 'chunkOverlap',
                type: 'number',
+                description: 'Number of characters to overlap between chunks. Default is 200.',
+                default: 200,
                optional: true
            }
        ]
@@ -27,6 +27,7 @@ class RecursiveCharacterTextSplitter_TextSplitters implements INode {
                label: 'Chunk Size',
                name: 'chunkSize',
                type: 'number',
+                description: 'Number of characters in each chunk. Default is 1000.',
                default: 1000,
                optional: true
            },
@@ -34,6 +35,8 @@ class RecursiveCharacterTextSplitter_TextSplitters implements INode {
                label: 'Chunk Overlap',
                name: 'chunkOverlap',
                type: 'number',
+                description: 'Number of characters to overlap between chunks. Default is 200.',
+                default: 200,
                optional: true
            },
            {
@@ -56,6 +56,7 @@ class TokenTextSplitter_TextSplitters implements INode {
                label: 'Chunk Size',
                name: 'chunkSize',
                type: 'number',
+                description: 'Number of characters in each chunk. Default is 1000.',
                default: 1000,
                optional: true
            },
@@ -63,6 +64,8 @@ class TokenTextSplitter_TextSplitters implements INode {
                label: 'Chunk Overlap',
                name: 'chunkOverlap',
                type: 'number',
+                description: 'Number of characters to overlap between chunks. Default is 200.',
+                default: 200,
                optional: true
            }
        ]