Feature/DocumentStore (#2106)

* datasource: initial commit * datasource: datasource details and chunks * datasource: Document Store Node * more changes * Document Store - Base functionality * Document Store Loader Component * Document Store Loader Component * before merging the modularity PR * after merging the modularity PR * preview mode * initial draft PR * fixes * minor updates and fixes * preview with loader and splitter * preview with credential * show stored chunks * preview update... * edit config * save, preview and other changes * save, preview and other changes * save, process and other changes * save, process and other changes * alpha1 - for internal testing * rerouting urls * bug fix on new leader create * pagination support for chunks * delete document store * Update pnpm-lock.yaml * doc store card view * Update store files to use updated storage functions, Document Store Table View and other changes * ui changes * add expanded chunk dialog, improve ui * change throw Error to InternalError * Bug Fixes and removal of subFolder, adding of view chunks for store * lint fixes * merge changes * DocumentStoreStatus component * ui changes for doc store * add remove metadata key field, add custom document loader * add chatflows used doc store chips * add types/interfaces to DocumentStore Services * document loader list dialog title bar color change * update interfaces * Whereused Chatflow Name and Added chunkNo to retain order of created chunks. * use typeorm order chunkNo, ui changes --------- Co-authored-by: Henry <hzj94@hotmail.com> Co-authored-by: Henry Heng <henryheng@flowiseai.com>
2026-06-22 09:01:09 +03:00 · 2024-05-06 19:53:27 +05:30
parent af4e28aa91
commit 40e36d1b39
91 changed files with 38713 additions and 32791 deletions
@@ -1,4 +1,5 @@
-import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
+import { omit } from 'lodash'
+import { ICommonObject, IDocument, INode, INodeData, INodeParams } from '../../../src/Interface'
 import {
    UnstructuredLoaderOptions,
    UnstructuredLoaderStrategy,
@@ -400,9 +401,21 @@ class UnstructuredFile_DocumentLoaders implements INode {
                default: '500'
            },
            {
-                label: 'Metadata',
+                label: 'Additional Metadata',
                name: 'metadata',
                type: 'json',
+                description: 'Additional metadata to be added to the extracted documents',
+                optional: true,
+                additionalParams: true
+            },
+            {
+                label: 'Omit Metadata Keys',
+                name: 'omitMetadataKeys',
+                type: 'string',
+                rows: 4,
+                description:
+                    'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma',
+                placeholder: 'key1, key2, key3.nestedKey1',
                optional: true,
                additionalParams: true
            }
@@ -429,6 +442,12 @@ class UnstructuredFile_DocumentLoaders implements INode {
        const combineUnderNChars = nodeData.inputs?.combineUnderNChars as number
        const newAfterNChars = nodeData.inputs?.newAfterNChars as number
        const maxCharacters = nodeData.inputs?.maxCharacters as number
+        const _omitMetadataKeys = nodeData.inputs?.omitMetadataKeys as string
+
+        let omitMetadataKeys: string[] = []
+        if (_omitMetadataKeys) {
+            omitMetadataKeys = _omitMetadataKeys.split(',').map((key) => key.trim())
+        }
        const fileBase64 = nodeData.inputs?.fileObject as string

        const obj: UnstructuredLoaderOptions = {
@@ -452,7 +471,7 @@ class UnstructuredFile_DocumentLoaders implements INode {
        const unstructuredAPIKey = getCredentialParam('unstructuredAPIKey', credentialData, nodeData)
        if (unstructuredAPIKey) obj.apiKey = unstructuredAPIKey

-        let docs: any[] = []
+        let docs: IDocument[] = []
        let files: string[] = []

        if (fileBase64) {
@@ -499,19 +518,25 @@ class UnstructuredFile_DocumentLoaders implements INode {
            const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata)
            docs = docs.map((doc) => ({
                ...doc,
-                metadata: {
-                    ...doc.metadata,
-                    ...parsedMetadata,
-                    [sourceIdKey]: doc.metadata[sourceIdKey] || sourceIdKey
-                }
+                metadata: omit(
+                    {
+                        ...doc.metadata,
+                        ...parsedMetadata,
+                        [sourceIdKey]: doc.metadata[sourceIdKey] || sourceIdKey
+                    },
+                    omitMetadataKeys
+                )
            }))
        } else {
            docs = docs.map((doc) => ({
                ...doc,
-                metadata: {
-                    ...doc.metadata,
-                    [sourceIdKey]: doc.metadata[sourceIdKey] || sourceIdKey
-                }
+                metadata: omit(
+                    {
+                        ...doc.metadata,
+                        [sourceIdKey]: doc.metadata[sourceIdKey] || sourceIdKey
+                    },
+                    omitMetadataKeys
+                )
            }))
        }