farcasclaudiu/Flowise
1
0
Fork 0
mirror of https://github.com/farcasclaudiu/Flowise.git synced 2026-06-28 13:00:56 +03:00
Code Issues Packages Projects Releases Wiki Activity

Feature/update upsert API (#3836)

Browse Source 
* update upsert API

* add fix for override files in upsert vector
This commit is contained in:
Henry Heng
2025-01-09 13:22:35 +00:00
committed by GitHub
parent 1ae78c2739
commit 8d266052ae
14 changed files with 766 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/server/src/Interface.DocumentStore.ts
+2
View File
@@ -70,6 +70,8 @@ export interface IDocumentStoreLoaderForPreview extends IDocumentStoreLoader {
export interface IDocumentStoreUpsertData {
docId: string
metadata?: string | object
replaceExisting?: boolean
loader?: {
name: string
config: ICommonObject
packages/server/src/controllers/documentstore/index.ts
+23 -1
View File
@@ -428,6 +428,27 @@ const generateDocStoreToolDesc = async (req: Request, res: Response, next: NextF
}
}
const getDocStoreConfigs = async (req: Request, res: Response, next: NextFunction) => {
try {
if (typeof req.params.id === 'undefined' || req.params.id === '') {
throw new InternalFlowiseError(
StatusCodes.PRECONDITION_FAILED,
`Error: documentStoreController.getDocStoreConfigs - storeId not provided!`
)
}
if (typeof req.params.loaderId === 'undefined' || req.params.loaderId === '') {
throw new InternalFlowiseError(
StatusCodes.PRECONDITION_FAILED,
`Error: documentStoreController.getDocStoreConfigs - doc loader Id not provided!`
)
}
const apiResponse = await documentStoreService.findDocStoreAvailableConfigs(req.params.id, req.params.loaderId)
return res.json(apiResponse)
} catch (error) {
next(error)
}
}
export default {
deleteDocumentStore,
createDocumentStore,
@@ -453,5 +474,6 @@ export default {
upsertDocStoreMiddleware,
refreshDocStoreMiddleware,
saveProcessingLoader,
generateDocStoreToolDesc
generateDocStoreToolDesc,
getDocStoreConfigs
}
packages/server/src/index.ts
+2 -21
View File
@@ -25,6 +25,7 @@ import { validateAPIKey } from './utils/validateKey'
import { IMetricsProvider } from './Interface.Metrics'
import { Prometheus } from './metrics/Prometheus'
import { OpenTelemetry } from './metrics/OpenTelemetry'
import { WHITELIST_URLS } from './utils/constants'
import 'global-agent/bootstrap'
declare global {
@@ -124,27 +125,7 @@ export class App {
next()
})
const whitelistURLs = [
'/api/v1/verify/apikey/',
'/api/v1/chatflows/apikey/',
'/api/v1/public-chatflows',
'/api/v1/public-chatbotConfig',
'/api/v1/prediction/',
'/api/v1/vector/upsert/',
'/api/v1/node-icon/',
'/api/v1/components-credentials-icon/',
'/api/v1/chatflows-streaming',
'/api/v1/chatflows-uploads',
'/api/v1/openai-assistants-file/download',
'/api/v1/feedback',
'/api/v1/leads',
'/api/v1/get-upload-file',
'/api/v1/ip',
'/api/v1/ping',
'/api/v1/version',
'/api/v1/attachments',
'/api/v1/metrics'
]
const whitelistURLs = WHITELIST_URLS
const URL_CASE_INSENSITIVE_REGEX: RegExp = /\/api\/v1\//i
const URL_CASE_SENSITIVE_REGEX: RegExp = /\/api\/v1\//
packages/server/src/routes/documentstore/index.ts
+2
View File
@@ -21,6 +21,8 @@ router.get('/store/:id', documentStoreController.getDocumentStoreById)
router.put('/store/:id', documentStoreController.updateDocumentStore)
// Delete documentStore
router.delete('/store/:id', documentStoreController.deleteDocumentStore)
// Get document store configs
router.get('/store-configs/:id/:loaderId', documentStoreController.getDocStoreConfigs)
/** Component Nodes = Document Store - Loaders */
// Get all loaders
packages/server/src/services/assistants/index.ts
+3 -17
View File
@@ -15,6 +15,7 @@ import { DocumentStore } from '../../database/entities/DocumentStore'
import { ICommonObject } from 'flowise-components'
import logger from '../../utils/logger'
import { ASSISTANT_PROMPT_GENERATOR } from '../../utils/prompt'
import { INPUT_PARAMS_TYPE } from '../../utils/constants'
const createAssistant = async (requestBody: any): Promise<Assistant> => {
try {
@@ -425,26 +426,11 @@ const getDocumentStores = async (): Promise<any> => {
const getTools = async (): Promise<any> => {
try {
const tools = await nodesService.getAllNodesForCategory('Tools')
const whitelistTypes = [
'asyncOptions',
'options',
'multiOptions',
'datagrid',
'string',
'number',
'boolean',
'password',
'json',
'code',
'date',
'file',
'folder',
'tabs'
]
// filter out those tools that input params type are not in the list
const filteredTools = tools.filter((tool) => {
const inputs = tool.inputs || []
return inputs.every((input) => whitelistTypes.includes(input.type))
return inputs.every((input) => INPUT_PARAMS_TYPE.includes(input.type))
})
return filteredTools
} catch (error) {
packages/server/src/services/documentstore/index.ts
+162 -3
View File
@@ -24,7 +24,8 @@ import {
IDocumentStoreRefreshData,
IDocumentStoreUpsertData,
IDocumentStoreWhereUsed,
INodeData
INodeData,
IOverrideConfig
} from '../../Interface'
import { DocumentStoreFileChunk } from '../../database/entities/DocumentStoreFileChunk'
import { v4 as uuidv4 } from 'uuid'
@@ -41,6 +42,7 @@ import { UpsertHistory } from '../../database/entities/UpsertHistory'
import { cloneDeep, omit } from 'lodash'
import { FLOWISE_COUNTER_STATUS, FLOWISE_METRIC_COUNTERS } from '../../Interface.Metrics'
import { DOCUMENTSTORE_TOOL_DESCRIPTION_PROMPT_GENERATOR } from '../../utils/prompt'
import { INPUT_PARAMS_TYPE } from '../../utils/constants'
const DOCUMENT_STORE_BASE_FOLDER = 'docustore'
@@ -1323,6 +1325,15 @@ const upsertDocStoreMiddleware = async (
) => {
const appServer = getRunningExpressApp()
const docId = data.docId
let metadata = {}
if (data.metadata) {
try {
metadata = typeof data.metadata === 'string' ? JSON.parse(data.metadata) : data.metadata
} catch (error) {
throw new InternalFlowiseError(StatusCodes.BAD_REQUEST, `Error: Invalid metadata`)
}
}
const replaceExisting = data.replaceExisting ?? false
const newLoader = typeof data.loader === 'string' ? JSON.parse(data.loader) : data.loader
const newSplitter = typeof data.splitter === 'string' ? JSON.parse(data.splitter) : data.splitter
const newVectorStore = typeof data.vectorStore === 'string' ? JSON.parse(data.vectorStore) : data.vectorStore
@@ -1479,6 +1490,13 @@ const upsertDocStoreMiddleware = async (
}
}
if (Object.keys(metadata).length > 0) {
loaderConfig = {
...loaderConfig,
metadata
}
}
// Step 4: Verification for must have components
if (!loaderName || !loaderId || !loaderConfig) {
throw new InternalFlowiseError(StatusCodes.INTERNAL_SERVER_ERROR, `Loader not configured`)
@@ -1503,7 +1521,7 @@ const upsertDocStoreMiddleware = async (
splitterConfig
}
if (isRefreshExisting) {
if (isRefreshExisting || replaceExisting) {
processData.id = docId
}
@@ -1629,6 +1647,146 @@ const generateDocStoreToolDesc = async (docStoreId: string, selectedChatModel: I
}
}
export const findDocStoreAvailableConfigs = async (storeId: string, docId: string) => {
// find the document store
const appServer = getRunningExpressApp()
const entity = await appServer.AppDataSource.getRepository(DocumentStore).findOneBy({ id: storeId })
if (!entity) {
throw new InternalFlowiseError(StatusCodes.NOT_FOUND, `Document store ${storeId} not found`)
}
const loaders = JSON.parse(entity.loaders)
const loader = loaders.find((ldr: IDocumentStoreLoader) => ldr.id === docId)
if (!loader) {
throw new InternalFlowiseError(StatusCodes.NOT_FOUND, `Document loader ${docId} not found`)
}
const nodes = []
const componentCredentials = appServer.nodesPool.componentCredentials
const loaderName = loader.loaderId
const loaderLabel = appServer.nodesPool.componentNodes[loaderName].label
const loaderInputs =
appServer.nodesPool.componentNodes[loaderName].inputs?.filter((input) => INPUT_PARAMS_TYPE.includes(input.type)) ?? []
nodes.push({
label: loaderLabel,
nodeId: `${loaderName}_0`,
inputParams: loaderInputs
})
const splitterName = loader.splitterId
if (splitterName) {
const splitterLabel = appServer.nodesPool.componentNodes[splitterName].label
const splitterInputs =
appServer.nodesPool.componentNodes[splitterName].inputs?.filter((input) => INPUT_PARAMS_TYPE.includes(input.type)) ?? []
nodes.push({
label: splitterLabel,
nodeId: `${splitterName}_0`,
inputParams: splitterInputs
})
}
if (entity.vectorStoreConfig) {
const vectorStoreName = JSON.parse(entity.vectorStoreConfig || '{}').name
const vectorStoreLabel = appServer.nodesPool.componentNodes[vectorStoreName].label
const vectorStoreInputs =
appServer.nodesPool.componentNodes[vectorStoreName].inputs?.filter((input) => INPUT_PARAMS_TYPE.includes(input.type)) ?? []
nodes.push({
label: vectorStoreLabel,
nodeId: `${vectorStoreName}_0`,
inputParams: vectorStoreInputs
})
}
if (entity.embeddingConfig) {
const embeddingName = JSON.parse(entity.embeddingConfig || '{}').name
const embeddingLabel = appServer.nodesPool.componentNodes[embeddingName].label
const embeddingInputs =
appServer.nodesPool.componentNodes[embeddingName].inputs?.filter((input) => INPUT_PARAMS_TYPE.includes(input.type)) ?? []
nodes.push({
label: embeddingLabel,
nodeId: `${embeddingName}_0`,
inputParams: embeddingInputs
})
}
if (entity.recordManagerConfig) {
const recordManagerName = JSON.parse(entity.recordManagerConfig || '{}').name
const recordManagerLabel = appServer.nodesPool.componentNodes[recordManagerName].label
const recordManagerInputs =
appServer.nodesPool.componentNodes[recordManagerName].inputs?.filter((input) => INPUT_PARAMS_TYPE.includes(input.type)) ?? []
nodes.push({
label: recordManagerLabel,
nodeId: `${recordManagerName}_0`,
inputParams: recordManagerInputs
})
}
const configs: IOverrideConfig[] = []
for (const node of nodes) {
const inputParams = node.inputParams
for (const inputParam of inputParams) {
let obj: IOverrideConfig
if (inputParam.type === 'file') {
obj = {
node: node.label,
nodeId: node.nodeId,
label: inputParam.label,
name: 'files',
type: inputParam.fileType ?? inputParam.type
}
} else if (inputParam.type === 'options') {
obj = {
node: node.label,
nodeId: node.nodeId,
label: inputParam.label,
name: inputParam.name,
type: inputParam.options
? inputParam.options
?.map((option) => {
return option.name
})
.join(', ')
: 'string'
}
} else if (inputParam.type === 'credential') {
// get component credential inputs
for (const name of inputParam.credentialNames ?? []) {
if (Object.prototype.hasOwnProperty.call(componentCredentials, name)) {
const inputs = componentCredentials[name]?.inputs ?? []
for (const input of inputs) {
obj = {
node: node.label,
nodeId: node.nodeId,
label: input.label,
name: input.name,
type: input.type === 'password' ? 'string' : input.type
}
configs.push(obj)
}
}
}
continue
} else {
obj = {
node: node.label,
nodeId: node.nodeId,
label: inputParam.label,
name: inputParam.name,
type: inputParam.type === 'password' ? 'string' : inputParam.type
}
}
if (!configs.some((config) => JSON.stringify(config) === JSON.stringify(obj))) {
configs.push(obj)
}
}
}
return configs
}
export default {
updateDocumentStoreUsage,
deleteDocumentStore,
@@ -1656,5 +1814,6 @@ export default {
updateVectorStoreConfigOnly,
upsertDocStoreMiddleware,
refreshDocStoreMiddleware,
generateDocStoreToolDesc
generateDocStoreToolDesc,
findDocStoreAvailableConfigs
}
packages/server/src/utils/constants.ts
+38
View File
@@ -0,0 +1,38 @@
export const WHITELIST_URLS = [
'/api/v1/verify/apikey/',
'/api/v1/chatflows/apikey/',
'/api/v1/public-chatflows',
'/api/v1/public-chatbotConfig',
'/api/v1/prediction/',
'/api/v1/vector/upsert/',
'/api/v1/node-icon/',
'/api/v1/components-credentials-icon/',
'/api/v1/chatflows-streaming',
'/api/v1/chatflows-uploads',
'/api/v1/openai-assistants-file/download',
'/api/v1/feedback',
'/api/v1/leads',
'/api/v1/get-upload-file',
'/api/v1/ip',
'/api/v1/ping',
'/api/v1/version',
'/api/v1/attachments',
'/api/v1/metrics'
]
export const INPUT_PARAMS_TYPE = [
'asyncOptions',
'options',
'multiOptions',
'datagrid',
'string',
'number',
'boolean',
'password',
'json',
'code',
'date',
'file',
'folder',
'tabs'
]
packages/server/src/utils/upsertVector.ts
+19 -6
View File
@@ -161,16 +161,29 @@ export const upsertVector = async (req: Request, isInternal: boolean = false) =>
const availableVariables = await appServer.AppDataSource.getRepository(Variable).find()
const { nodeOverrides, variableOverrides, apiOverrideStatus } = getAPIOverrideConfig(chatflow)
// For "files" input, add a new node override with the actual input name such as pdfFile, txtFile, etc.
// For "files" input, add a new node override with the actual input name such as pdfFile, txtFile, etc, to allow overriding the input
for (const nodeLabel in nodeOverrides) {
const params = nodeOverrides[nodeLabel]
const enabledFileParam = params.find((param) => param.enabled && param.name === 'files')
if (enabledFileParam) {
const fileInputFieldFromExt = mapExtToInputField(enabledFileParam.type)
nodeOverrides[nodeLabel].push({
...enabledFileParam,
name: fileInputFieldFromExt
})
if (enabledFileParam.type.includes(',')) {
const fileInputFieldsFromExt = enabledFileParam.type.split(',').map((fileType) => mapExtToInputField(fileType.trim()))
for (const fileInputFieldFromExt of fileInputFieldsFromExt) {
if (nodeOverrides[nodeLabel].some((param) => param.name === fileInputFieldFromExt)) {
continue
}
nodeOverrides[nodeLabel].push({
...enabledFileParam,
name: fileInputFieldFromExt
})
}
} else {
const fileInputFieldFromExt = mapExtToInputField(enabledFileParam.type)
nodeOverrides[nodeLabel].push({
...enabledFileParam,
name: fileInputFieldFromExt
})
}
}
}