mirror of
https://github.com/farcasclaudiu/Flowise.git
synced 2026-06-29 01:01:11 +03:00
Feature/DocumentStore (#2106)
* datasource: initial commit * datasource: datasource details and chunks * datasource: Document Store Node * more changes * Document Store - Base functionality * Document Store Loader Component * Document Store Loader Component * before merging the modularity PR * after merging the modularity PR * preview mode * initial draft PR * fixes * minor updates and fixes * preview with loader and splitter * preview with credential * show stored chunks * preview update... * edit config * save, preview and other changes * save, preview and other changes * save, process and other changes * save, process and other changes * alpha1 - for internal testing * rerouting urls * bug fix on new leader create * pagination support for chunks * delete document store * Update pnpm-lock.yaml * doc store card view * Update store files to use updated storage functions, Document Store Table View and other changes * ui changes * add expanded chunk dialog, improve ui * change throw Error to InternalError * Bug Fixes and removal of subFolder, adding of view chunks for store * lint fixes * merge changes * DocumentStoreStatus component * ui changes for doc store * add remove metadata key field, add custom document loader * add chatflows used doc store chips * add types/interfaces to DocumentStore Services * document loader list dialog title bar color change * update interfaces * Whereused Chatflow Name and Added chunkNo to retain order of created chunks. * use typeorm order chunkNo, ui changes --------- Co-authored-by: Henry <hzj94@hotmail.com> Co-authored-by: Henry Heng <henryheng@flowiseai.com>
This commit is contained in:
Vinod Kiran
+163
@@ -0,0 +1,163 @@
|
||||
import { ICommonObject, IDatabaseEntity, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
|
||||
import { NodeVM } from 'vm2'
|
||||
import { DataSource } from 'typeorm'
|
||||
import { availableDependencies, defaultAllowBuiltInDep, getVars, handleEscapeCharacters, prepareSandboxVars } from '../../../src/utils'
|
||||
|
||||
class CustomDocumentLoader_DocumentLoaders implements INode {
|
||||
label: string
|
||||
name: string
|
||||
version: number
|
||||
description: string
|
||||
type: string
|
||||
icon: string
|
||||
category: string
|
||||
badge: string
|
||||
baseClasses: string[]
|
||||
inputs: INodeParams[]
|
||||
outputs: INodeOutputsValue[]
|
||||
|
||||
constructor() {
|
||||
this.label = 'Custom Document Loader'
|
||||
this.name = 'customDocumentLoader'
|
||||
this.version = 1.0
|
||||
this.type = 'Document'
|
||||
this.icon = 'customDocLoader.svg'
|
||||
this.category = 'Document Loaders'
|
||||
this.badge = 'NEW'
|
||||
this.description = `Custom function for loading documents`
|
||||
this.baseClasses = [this.type]
|
||||
this.inputs = [
|
||||
{
|
||||
label: 'Input Variables',
|
||||
name: 'functionInputVariables',
|
||||
description: 'Input variables can be used in the function with prefix $. For example: $var',
|
||||
type: 'json',
|
||||
optional: true,
|
||||
acceptVariable: true,
|
||||
list: true
|
||||
},
|
||||
{
|
||||
label: 'Javascript Function',
|
||||
name: 'javascriptFunction',
|
||||
type: 'code',
|
||||
description: `Must return an array of document objects containing metadata and pageContent if "Document" is selected in the output. If "Text" is selected in the output, it must return a string.`,
|
||||
placeholder: `return [
|
||||
{
|
||||
pageContent: 'Document Content',
|
||||
metadata: {
|
||||
title: 'Document Title',
|
||||
}
|
||||
}
|
||||
]`
|
||||
}
|
||||
]
|
||||
this.outputs = [
|
||||
{
|
||||
label: 'Document',
|
||||
name: 'document',
|
||||
description: 'Array of document objects containing metadata and pageContent',
|
||||
baseClasses: [...this.baseClasses, 'json']
|
||||
},
|
||||
{
|
||||
label: 'Text',
|
||||
name: 'text',
|
||||
description: 'Concatenated string from pageContent of documents',
|
||||
baseClasses: ['string', 'json']
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
async init(nodeData: INodeData, input: string, options: ICommonObject): Promise<any> {
|
||||
const output = nodeData.outputs?.output as string
|
||||
const javascriptFunction = nodeData.inputs?.javascriptFunction as string
|
||||
const functionInputVariablesRaw = nodeData.inputs?.functionInputVariables
|
||||
const appDataSource = options.appDataSource as DataSource
|
||||
const databaseEntities = options.databaseEntities as IDatabaseEntity
|
||||
|
||||
const variables = await getVars(appDataSource, databaseEntities, nodeData)
|
||||
const flow = {
|
||||
chatflowId: options.chatflowid,
|
||||
sessionId: options.sessionId,
|
||||
chatId: options.chatId,
|
||||
input
|
||||
}
|
||||
|
||||
let inputVars: ICommonObject = {}
|
||||
if (functionInputVariablesRaw) {
|
||||
try {
|
||||
inputVars =
|
||||
typeof functionInputVariablesRaw === 'object' ? functionInputVariablesRaw : JSON.parse(functionInputVariablesRaw)
|
||||
} catch (exception) {
|
||||
throw new Error('Invalid JSON in the Custom Document Loader Input Variables: ' + exception)
|
||||
}
|
||||
}
|
||||
|
||||
// Some values might be a stringified JSON, parse it
|
||||
for (const key in inputVars) {
|
||||
let value = inputVars[key]
|
||||
if (typeof value === 'string') {
|
||||
value = handleEscapeCharacters(value, true)
|
||||
if (value.startsWith('{') && value.endsWith('}')) {
|
||||
try {
|
||||
value = JSON.parse(value)
|
||||
} catch (e) {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
inputVars[key] = value
|
||||
}
|
||||
}
|
||||
|
||||
let sandbox: any = { $input: input }
|
||||
sandbox['$vars'] = prepareSandboxVars(variables)
|
||||
sandbox['$flow'] = flow
|
||||
|
||||
if (Object.keys(inputVars).length) {
|
||||
for (const item in inputVars) {
|
||||
sandbox[`$${item}`] = inputVars[item]
|
||||
}
|
||||
}
|
||||
|
||||
const builtinDeps = process.env.TOOL_FUNCTION_BUILTIN_DEP
|
||||
? defaultAllowBuiltInDep.concat(process.env.TOOL_FUNCTION_BUILTIN_DEP.split(','))
|
||||
: defaultAllowBuiltInDep
|
||||
const externalDeps = process.env.TOOL_FUNCTION_EXTERNAL_DEP ? process.env.TOOL_FUNCTION_EXTERNAL_DEP.split(',') : []
|
||||
const deps = availableDependencies.concat(externalDeps)
|
||||
|
||||
const nodeVMOptions = {
|
||||
console: 'inherit',
|
||||
sandbox,
|
||||
require: {
|
||||
external: { modules: deps },
|
||||
builtin: builtinDeps
|
||||
}
|
||||
} as any
|
||||
|
||||
const vm = new NodeVM(nodeVMOptions)
|
||||
try {
|
||||
const response = await vm.run(`module.exports = async function() {${javascriptFunction}}()`, __dirname)
|
||||
|
||||
if (output === 'document' && Array.isArray(response)) {
|
||||
if (response.length === 0) return response
|
||||
if (
|
||||
response[0].pageContent &&
|
||||
typeof response[0].pageContent === 'string' &&
|
||||
response[0].metadata &&
|
||||
typeof response[0].metadata === 'object'
|
||||
)
|
||||
return response
|
||||
throw new Error('Document object must contain pageContent and metadata')
|
||||
}
|
||||
|
||||
if (output === 'text' && typeof response === 'string') {
|
||||
return handleEscapeCharacters(response, false)
|
||||
}
|
||||
|
||||
return response
|
||||
} catch (e) {
|
||||
throw new Error(e)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { nodeClass: CustomDocumentLoader_DocumentLoaders }
|
||||
@@ -0,0 +1 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="icon icon-tabler icons-tabler-outline icon-tabler-writing"><path stroke="none" d="M0 0h24v24H0z" fill="none"/><path d="M20 17v-12c0 -1.121 -.879 -2 -2 -2s-2 .879 -2 2v12l2 2l2 -2z" /><path d="M16 7h4" /><path d="M18 19h-13a2 2 0 1 1 0 -4h4a2 2 0 1 0 0 -4h-3" /></svg>
|
||||
|
After Width: | Height: | Size: 465 B |
Reference in New Issue
Block a user