farcasclaudiu/Flowise
1
0
Fork 0
mirror of https://github.com/farcasclaudiu/Flowise.git synced 2026-06-28 15:00:57 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into chore/Upgrade-LC-version

Browse Source
This commit is contained in:
Henry
2024-02-19 17:39:32 +08:00
parent 17a27d92a5 f0057739c9
commit 5a45a99620
94 changed files with 2265 additions and 702 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/components/nodes/chatmodels/ChatOpenAI/ChatOpenAI.ts
+4
View File
@@ -59,6 +59,10 @@ class ChatOpenAI_ChatModels implements INode {
label: 'gpt-4-1106-preview',
name: 'gpt-4-1106-preview'
},
{
label: 'gpt-4-1106-vision-preview',
name: 'gpt-4-1106-vision-preview'
},
{
label: 'gpt-4-vision-preview',
name: 'gpt-4-vision-preview'
packages/components/nodes/documentloaders/Cheerio/Cheerio.ts
+4 -2
View File
@@ -126,7 +126,9 @@ class Cheerio_DocumentLoaders implements INode {
let docs = []
if (relativeLinksMethod) {
if (process.env.DEBUG === 'true') options.logger.info(`Start ${relativeLinksMethod}`)
if (!limit) limit = 10
// if limit is 0 we don't want it to default to 10 so we check explicitly for null or undefined
// so when limit is 0 we can fetch all the links
if (limit === null || limit === undefined) limit = 10
else if (limit < 0) throw new Error('Limit cannot be less than 0')
const pages: string[] =
selectedLinks && selectedLinks.length > 0
@@ -143,7 +145,7 @@ class Cheerio_DocumentLoaders implements INode {
} else if (selectedLinks && selectedLinks.length > 0) {
if (process.env.DEBUG === 'true')
options.logger.info(`pages: ${JSON.stringify(selectedLinks)}, length: ${selectedLinks.length}`)
for (const page of selectedLinks) {
for (const page of selectedLinks.slice(0, limit)) {
docs.push(...(await cheerioLoader(page)))
}
} else {
packages/components/nodes/documentloaders/PlainText/PlainText.ts
+3 -1
View File
@@ -51,11 +51,13 @@ class PlainText_DocumentLoaders implements INode {
{
label: 'Document',
name: 'document',
baseClasses: this.baseClasses
description: 'Array of document objects containing metadata and pageContent',
baseClasses: [...this.baseClasses, 'json']
},
{
label: 'Text',
name: 'text',
description: 'Concatenated string from pageContent of documents',
baseClasses: ['string', 'json']
}
]
packages/components/nodes/documentloaders/Playwright/Playwright.ts
+4 -2
View File
@@ -167,7 +167,9 @@ class Playwright_DocumentLoaders implements INode {
let docs = []
if (relativeLinksMethod) {
if (process.env.DEBUG === 'true') options.logger.info(`Start ${relativeLinksMethod}`)
if (!limit) limit = 10
// if limit is 0 we don't want it to default to 10 so we check explicitly for null or undefined
// so when limit is 0 we can fetch all the links
if (limit === null || limit === undefined) limit = 10
else if (limit < 0) throw new Error('Limit cannot be less than 0')
const pages: string[] =
selectedLinks && selectedLinks.length > 0
@@ -184,7 +186,7 @@ class Playwright_DocumentLoaders implements INode {
} else if (selectedLinks && selectedLinks.length > 0) {
if (process.env.DEBUG === 'true')
options.logger.info(`pages: ${JSON.stringify(selectedLinks)}, length: ${selectedLinks.length}`)
for (const page of selectedLinks) {
for (const page of selectedLinks.slice(0, limit)) {
docs.push(...(await playwrightLoader(page)))
}
} else {
packages/components/nodes/documentloaders/Puppeteer/Puppeteer.ts
+4 -2
View File
@@ -168,7 +168,9 @@ class Puppeteer_DocumentLoaders implements INode {
let docs = []
if (relativeLinksMethod) {
if (process.env.DEBUG === 'true') options.logger.info(`Start ${relativeLinksMethod}`)
if (!limit) limit = 10
// if limit is 0 we don't want it to default to 10 so we check explicitly for null or undefined
// so when limit is 0 we can fetch all the links
if (limit === null || limit === undefined) limit = 10
else if (limit < 0) throw new Error('Limit cannot be less than 0')
const pages: string[] =
selectedLinks && selectedLinks.length > 0
@@ -185,7 +187,7 @@ class Puppeteer_DocumentLoaders implements INode {
} else if (selectedLinks && selectedLinks.length > 0) {
if (process.env.DEBUG === 'true')
options.logger.info(`pages: ${JSON.stringify(selectedLinks)}, length: ${selectedLinks.length}`)
for (const page of selectedLinks) {
for (const page of selectedLinks.slice(0, limit)) {
docs.push(...(await puppeteerLoader(page)))
}
} else {
packages/components/nodes/documentloaders/Text/Text.ts
+3 -1
View File
@@ -51,11 +51,13 @@ class Text_DocumentLoaders implements INode {
{
label: 'Document',
name: 'document',
baseClasses: this.baseClasses
description: 'Array of document objects containing metadata and pageContent',
baseClasses: [...this.baseClasses, 'json']
},
{
label: 'Text',
name: 'text',
description: 'Concatenated string from pageContent of documents',
baseClasses: ['string', 'json']
}
]
packages/components/nodes/documentloaders/VectorStoreToDocument/VectorStoreToDocument.ts
+2
View File
@@ -51,11 +51,13 @@ class VectorStoreToDocument_DocumentLoaders implements INode {
{
label: 'Document',
name: 'document',
description: 'Array of document objects containing metadata and pageContent',
baseClasses: [...this.baseClasses, 'json']
},
{
label: 'Text',
name: 'text',
description: 'Concatenated string from pageContent of documents',
baseClasses: ['string', 'json']
}
]
packages/components/nodes/engine/QueryEngine/QueryEngine.ts
+50 -44
View File
@@ -28,12 +28,12 @@ class QueryEngine_LlamaIndex implements INode {
constructor(fields?: { sessionId?: string }) {
this.label = 'Query Engine'
this.name = 'queryEngine'
this.version = 1.0
this.version = 2.0
this.type = 'QueryEngine'
this.icon = 'query-engine.png'
this.category = 'Engine'
this.description = 'Simple query engine built to answer question over your data, without memory'
this.baseClasses = [this.type]
this.baseClasses = [this.type, 'BaseQueryEngine']
this.tags = ['LlamaIndex']
this.inputs = [
{
@@ -59,52 +59,13 @@ class QueryEngine_LlamaIndex implements INode {
this.sessionId = fields?.sessionId
}
async init(): Promise<any> {
return null
async init(nodeData: INodeData): Promise<any> {
return prepareEngine(nodeData)
}
async run(nodeData: INodeData, input: string, options: ICommonObject): Promise<string | object> {
const returnSourceDocuments = nodeData.inputs?.returnSourceDocuments as boolean
const vectorStoreRetriever = nodeData.inputs?.vectorStoreRetriever
const responseSynthesizerObj = nodeData.inputs?.responseSynthesizer
let queryEngine = new RetrieverQueryEngine(vectorStoreRetriever)
if (responseSynthesizerObj) {
if (responseSynthesizerObj.type === 'TreeSummarize') {
const responseSynthesizer = new ResponseSynthesizer({
responseBuilder: new TreeSummarize(vectorStoreRetriever.serviceContext, responseSynthesizerObj.textQAPromptTemplate),
serviceContext: vectorStoreRetriever.serviceContext
})
queryEngine = new RetrieverQueryEngine(vectorStoreRetriever, responseSynthesizer)
} else if (responseSynthesizerObj.type === 'CompactAndRefine') {
const responseSynthesizer = new ResponseSynthesizer({
responseBuilder: new CompactAndRefine(
vectorStoreRetriever.serviceContext,
responseSynthesizerObj.textQAPromptTemplate,
responseSynthesizerObj.refinePromptTemplate
),
serviceContext: vectorStoreRetriever.serviceContext
})
queryEngine = new RetrieverQueryEngine(vectorStoreRetriever, responseSynthesizer)
} else if (responseSynthesizerObj.type === 'Refine') {
const responseSynthesizer = new ResponseSynthesizer({
responseBuilder: new Refine(
vectorStoreRetriever.serviceContext,
responseSynthesizerObj.textQAPromptTemplate,
responseSynthesizerObj.refinePromptTemplate
),
serviceContext: vectorStoreRetriever.serviceContext
})
queryEngine = new RetrieverQueryEngine(vectorStoreRetriever, responseSynthesizer)
} else if (responseSynthesizerObj.type === 'SimpleResponseBuilder') {
const responseSynthesizer = new ResponseSynthesizer({
responseBuilder: new SimpleResponseBuilder(vectorStoreRetriever.serviceContext),
serviceContext: vectorStoreRetriever.serviceContext
})
queryEngine = new RetrieverQueryEngine(vectorStoreRetriever, responseSynthesizer)
}
}
const queryEngine = prepareEngine(nodeData)
let text = ''
let sourceDocuments: ICommonObject[] = []
@@ -140,4 +101,49 @@ class QueryEngine_LlamaIndex implements INode {
}
}
const prepareEngine = (nodeData: INodeData) => {
const vectorStoreRetriever = nodeData.inputs?.vectorStoreRetriever
const responseSynthesizerObj = nodeData.inputs?.responseSynthesizer
let queryEngine = new RetrieverQueryEngine(vectorStoreRetriever)
if (responseSynthesizerObj) {
if (responseSynthesizerObj.type === 'TreeSummarize') {
const responseSynthesizer = new ResponseSynthesizer({
responseBuilder: new TreeSummarize(vectorStoreRetriever.serviceContext, responseSynthesizerObj.textQAPromptTemplate),
serviceContext: vectorStoreRetriever.serviceContext
})
queryEngine = new RetrieverQueryEngine(vectorStoreRetriever, responseSynthesizer)
} else if (responseSynthesizerObj.type === 'CompactAndRefine') {
const responseSynthesizer = new ResponseSynthesizer({
responseBuilder: new CompactAndRefine(
vectorStoreRetriever.serviceContext,
responseSynthesizerObj.textQAPromptTemplate,
responseSynthesizerObj.refinePromptTemplate
),
serviceContext: vectorStoreRetriever.serviceContext
})
queryEngine = new RetrieverQueryEngine(vectorStoreRetriever, responseSynthesizer)
} else if (responseSynthesizerObj.type === 'Refine') {
const responseSynthesizer = new ResponseSynthesizer({
responseBuilder: new Refine(
vectorStoreRetriever.serviceContext,
responseSynthesizerObj.textQAPromptTemplate,
responseSynthesizerObj.refinePromptTemplate
),
serviceContext: vectorStoreRetriever.serviceContext
})
queryEngine = new RetrieverQueryEngine(vectorStoreRetriever, responseSynthesizer)
} else if (responseSynthesizerObj.type === 'SimpleResponseBuilder') {
const responseSynthesizer = new ResponseSynthesizer({
responseBuilder: new SimpleResponseBuilder(vectorStoreRetriever.serviceContext),
serviceContext: vectorStoreRetriever.serviceContext
})
queryEngine = new RetrieverQueryEngine(vectorStoreRetriever, responseSynthesizer)
}
}
return queryEngine
}
module.exports = { nodeClass: QueryEngine_LlamaIndex }
packages/components/nodes/engine/SubQuestionQueryEngine/SubQuestionQueryEngine.ts
+83 -77
View File
@@ -33,13 +33,13 @@ class SubQuestionQueryEngine_LlamaIndex implements INode {
constructor(fields?: { sessionId?: string }) {
this.label = 'Sub Question Query Engine'
this.name = 'subQuestionQueryEngine'
this.version = 1.0
this.version = 2.0
this.type = 'SubQuestionQueryEngine'
this.icon = 'subQueryEngine.svg'
this.category = 'Engine'
this.description =
'Breaks complex query into sub questions for each relevant data source, then gather all the intermediate reponses and synthesizes a final response'
this.baseClasses = [this.type]
this.baseClasses = [this.type, 'BaseQueryEngine']
this.tags = ['LlamaIndex']
this.inputs = [
{
@@ -76,85 +76,13 @@ class SubQuestionQueryEngine_LlamaIndex implements INode {
this.sessionId = fields?.sessionId
}
async init(): Promise<any> {
return null
async init(nodeData: INodeData): Promise<any> {
return prepareEngine(nodeData)
}
async run(nodeData: INodeData, input: string, options: ICommonObject): Promise<string | object> {
const returnSourceDocuments = nodeData.inputs?.returnSourceDocuments as boolean
const embeddings = nodeData.inputs?.embeddings as BaseEmbedding
const model = nodeData.inputs?.model
const serviceContext = serviceContextFromDefaults({
llm: model,
embedModel: embeddings
})
let queryEngineTools = nodeData.inputs?.queryEngineTools as QueryEngineTool[]
queryEngineTools = flatten(queryEngineTools)
let queryEngine = SubQuestionQueryEngine.fromDefaults({
serviceContext,
queryEngineTools,
questionGen: new LLMQuestionGenerator({ llm: model })
})
const responseSynthesizerObj = nodeData.inputs?.responseSynthesizer
if (responseSynthesizerObj) {
if (responseSynthesizerObj.type === 'TreeSummarize') {
const responseSynthesizer = new ResponseSynthesizer({
responseBuilder: new TreeSummarize(serviceContext, responseSynthesizerObj.textQAPromptTemplate),
serviceContext
})
queryEngine = SubQuestionQueryEngine.fromDefaults({
responseSynthesizer,
serviceContext,
queryEngineTools,
questionGen: new LLMQuestionGenerator({ llm: model })
})
} else if (responseSynthesizerObj.type === 'CompactAndRefine') {
const responseSynthesizer = new ResponseSynthesizer({
responseBuilder: new CompactAndRefine(
serviceContext,
responseSynthesizerObj.textQAPromptTemplate,
responseSynthesizerObj.refinePromptTemplate
),
serviceContext
})
queryEngine = SubQuestionQueryEngine.fromDefaults({
responseSynthesizer,
serviceContext,
queryEngineTools,
questionGen: new LLMQuestionGenerator({ llm: model })
})
} else if (responseSynthesizerObj.type === 'Refine') {
const responseSynthesizer = new ResponseSynthesizer({
responseBuilder: new Refine(
serviceContext,
responseSynthesizerObj.textQAPromptTemplate,
responseSynthesizerObj.refinePromptTemplate
),
serviceContext
})
queryEngine = SubQuestionQueryEngine.fromDefaults({
responseSynthesizer,
serviceContext,
queryEngineTools,
questionGen: new LLMQuestionGenerator({ llm: model })
})
} else if (responseSynthesizerObj.type === 'SimpleResponseBuilder') {
const responseSynthesizer = new ResponseSynthesizer({
responseBuilder: new SimpleResponseBuilder(serviceContext),
serviceContext
})
queryEngine = SubQuestionQueryEngine.fromDefaults({
responseSynthesizer,
serviceContext,
queryEngineTools,
questionGen: new LLMQuestionGenerator({ llm: model })
})
}
}
const queryEngine = prepareEngine(nodeData)
let text = ''
let sourceDocuments: ICommonObject[] = []
@@ -190,4 +118,82 @@ class SubQuestionQueryEngine_LlamaIndex implements INode {
}
}
const prepareEngine = (nodeData: INodeData) => {
const embeddings = nodeData.inputs?.embeddings as BaseEmbedding
const model = nodeData.inputs?.model
const serviceContext = serviceContextFromDefaults({
llm: model,
embedModel: embeddings
})
let queryEngineTools = nodeData.inputs?.queryEngineTools as QueryEngineTool[]
queryEngineTools = flatten(queryEngineTools)
let queryEngine = SubQuestionQueryEngine.fromDefaults({
serviceContext,
queryEngineTools,
questionGen: new LLMQuestionGenerator({ llm: model })
})
const responseSynthesizerObj = nodeData.inputs?.responseSynthesizer
if (responseSynthesizerObj) {
if (responseSynthesizerObj.type === 'TreeSummarize') {
const responseSynthesizer = new ResponseSynthesizer({
responseBuilder: new TreeSummarize(serviceContext, responseSynthesizerObj.textQAPromptTemplate),
serviceContext
})
queryEngine = SubQuestionQueryEngine.fromDefaults({
responseSynthesizer,
serviceContext,
queryEngineTools,
questionGen: new LLMQuestionGenerator({ llm: model })
})
} else if (responseSynthesizerObj.type === 'CompactAndRefine') {
const responseSynthesizer = new ResponseSynthesizer({
responseBuilder: new CompactAndRefine(
serviceContext,
responseSynthesizerObj.textQAPromptTemplate,
responseSynthesizerObj.refinePromptTemplate
),
serviceContext
})
queryEngine = SubQuestionQueryEngine.fromDefaults({
responseSynthesizer,
serviceContext,
queryEngineTools,
questionGen: new LLMQuestionGenerator({ llm: model })
})
} else if (responseSynthesizerObj.type === 'Refine') {
const responseSynthesizer = new ResponseSynthesizer({
responseBuilder: new Refine(
serviceContext,
responseSynthesizerObj.textQAPromptTemplate,
responseSynthesizerObj.refinePromptTemplate
),
serviceContext
})
queryEngine = SubQuestionQueryEngine.fromDefaults({
responseSynthesizer,
serviceContext,
queryEngineTools,
questionGen: new LLMQuestionGenerator({ llm: model })
})
} else if (responseSynthesizerObj.type === 'SimpleResponseBuilder') {
const responseSynthesizer = new ResponseSynthesizer({
responseBuilder: new SimpleResponseBuilder(serviceContext),
serviceContext
})
queryEngine = SubQuestionQueryEngine.fromDefaults({
responseSynthesizer,
serviceContext,
queryEngineTools,
questionGen: new LLMQuestionGenerator({ llm: model })
})
}
}
return queryEngine
}
module.exports = { nodeClass: SubQuestionQueryEngine_LlamaIndex }
packages/components/nodes/memory/DynamoDb/DynamoDb.ts
+24 -11
View File
@@ -117,7 +117,10 @@ const initalizeDynamoDB = async (nodeData: INodeData, options: ICommonObject): P
memoryKey: memoryKey ?? 'chat_history',
chatHistory: dynamoDb,
sessionId,
dynamodbClient: client
dynamodbClient: client,
tableName,
partitionKey,
dynamoKey: { [partitionKey]: { S: sessionId } }
})
return memory
}
@@ -125,6 +128,9 @@ const initalizeDynamoDB = async (nodeData: INodeData, options: ICommonObject): P
interface BufferMemoryExtendedInput {
dynamodbClient: DynamoDBClient
sessionId: string
tableName: string
partitionKey: string
dynamoKey: Record<string, AttributeValue>
}
interface DynamoDBSerializedChatMessage {
@@ -142,6 +148,10 @@ interface DynamoDBSerializedChatMessage {
}
class BufferMemoryExtended extends FlowiseMemory implements MemoryMethods {
private tableName = ''
private partitionKey = ''
private dynamoKey: Record<string, AttributeValue>
private messageAttributeName: string
sessionId = ''
dynamodbClient: DynamoDBClient
@@ -149,11 +159,14 @@ class BufferMemoryExtended extends FlowiseMemory implements MemoryMethods {
super(fields)
this.sessionId = fields.sessionId
this.dynamodbClient = fields.dynamodbClient
this.tableName = fields.tableName
this.partitionKey = fields.partitionKey
this.dynamoKey = fields.dynamoKey
}
overrideDynamoKey(overrideSessionId = '') {
const existingDynamoKey = (this as any).dynamoKey
const partitionKey = (this as any).partitionKey
const existingDynamoKey = this.dynamoKey
const partitionKey = this.partitionKey
let newDynamoKey: Record<string, AttributeValue> = {}
@@ -209,9 +222,9 @@ class BufferMemoryExtended extends FlowiseMemory implements MemoryMethods {
async getChatMessages(overrideSessionId = '', returnBaseMessages = false): Promise<IMessage[] | BaseMessage[]> {
if (!this.dynamodbClient) return []
const dynamoKey = overrideSessionId ? this.overrideDynamoKey(overrideSessionId) : (this as any).dynamoKey
const tableName = (this as any).tableName
const messageAttributeName = (this as any).messageAttributeName
const dynamoKey = overrideSessionId ? this.overrideDynamoKey(overrideSessionId) : this.dynamoKey
const tableName = this.tableName
const messageAttributeName = this.messageAttributeName
const params: GetItemCommandInput = {
TableName: tableName,
@@ -236,9 +249,9 @@ class BufferMemoryExtended extends FlowiseMemory implements MemoryMethods {
async addChatMessages(msgArray: { text: string; type: MessageType }[], overrideSessionId = ''): Promise<void> {
if (!this.dynamodbClient) return
const dynamoKey = overrideSessionId ? this.overrideDynamoKey(overrideSessionId) : (this as any).dynamoKey
const tableName = (this as any).tableName
const messageAttributeName = (this as any).messageAttributeName
const dynamoKey = overrideSessionId ? this.overrideDynamoKey(overrideSessionId) : this.dynamoKey
const tableName = this.tableName
const messageAttributeName = this.messageAttributeName
const input = msgArray.find((msg) => msg.type === 'userMessage')
const output = msgArray.find((msg) => msg.type === 'apiMessage')
@@ -259,8 +272,8 @@ class BufferMemoryExtended extends FlowiseMemory implements MemoryMethods {
async clearChatMessages(overrideSessionId = ''): Promise<void> {
if (!this.dynamodbClient) return
const dynamoKey = overrideSessionId ? this.overrideDynamoKey(overrideSessionId) : (this as any).dynamoKey
const tableName = (this as any).tableName
const dynamoKey = overrideSessionId ? this.overrideDynamoKey(overrideSessionId) : this.dynamoKey
const tableName = this.tableName
const params: DeleteItemCommandInput = {
TableName: tableName,
packages/components/nodes/memory/MongoDBMemory/MongoDBMemory.ts
+3 -3
View File
@@ -154,7 +154,7 @@ class BufferMemoryExtended extends FlowiseMemory implements MemoryMethods {
async getChatMessages(overrideSessionId = '', returnBaseMessages = false): Promise<IMessage[] | BaseMessage[]> {
if (!this.collection) return []
const id = overrideSessionId ?? this.sessionId
const id = overrideSessionId ? overrideSessionId : this.sessionId
const document = await this.collection.findOne({ sessionId: id })
const messages = document?.messages || []
const baseMessages = messages.map(mapStoredMessageToChatMessage)
@@ -164,7 +164,7 @@ class BufferMemoryExtended extends FlowiseMemory implements MemoryMethods {
async addChatMessages(msgArray: { text: string; type: MessageType }[], overrideSessionId = ''): Promise<void> {
if (!this.collection) return
const id = overrideSessionId ?? this.sessionId
const id = overrideSessionId ? overrideSessionId : this.sessionId
const input = msgArray.find((msg) => msg.type === 'userMessage')
const output = msgArray.find((msg) => msg.type === 'apiMessage')
@@ -196,7 +196,7 @@ class BufferMemoryExtended extends FlowiseMemory implements MemoryMethods {
async clearChatMessages(overrideSessionId = ''): Promise<void> {
if (!this.collection) return
const id = overrideSessionId ?? this.sessionId
const id = overrideSessionId ? overrideSessionId : this.sessionId
await this.collection.deleteOne({ sessionId: id })
await this.clear()
}
packages/components/nodes/memory/MotorheadMemory/MotorheadMemory.ts
+3 -3
View File
@@ -141,7 +141,7 @@ class MotorheadMemoryExtended extends MotorheadMemory implements MemoryMethods {
}
async getChatMessages(overrideSessionId = '', returnBaseMessages = false): Promise<IMessage[] | BaseMessage[]> {
const id = overrideSessionId ?? this.sessionId
const id = overrideSessionId ? overrideSessionId : this.sessionId
try {
const resp = await this.caller.call(fetch, `${this.url}/sessions/${id}/memory`, {
//@ts-ignore
@@ -172,7 +172,7 @@ class MotorheadMemoryExtended extends MotorheadMemory implements MemoryMethods {
}
async addChatMessages(msgArray: { text: string; type: MessageType }[], overrideSessionId = ''): Promise<void> {
const id = overrideSessionId ?? this.sessionId
const id = overrideSessionId ? overrideSessionId : this.sessionId
const input = msgArray.find((msg) => msg.type === 'userMessage')
const output = msgArray.find((msg) => msg.type === 'apiMessage')
const inputValues = { [this.inputKey ?? 'input']: input?.text }
@@ -182,7 +182,7 @@ class MotorheadMemoryExtended extends MotorheadMemory implements MemoryMethods {
}
async clearChatMessages(overrideSessionId = ''): Promise<void> {
const id = overrideSessionId ?? this.sessionId
const id = overrideSessionId ? overrideSessionId : this.sessionId
await this.clear(id)
}
}
packages/components/nodes/memory/RedisBackedChatMemory/RedisBackedChatMemory.ts
+3 -3
View File
@@ -189,7 +189,7 @@ class BufferMemoryExtended extends FlowiseMemory implements MemoryMethods {
async getChatMessages(overrideSessionId = '', returnBaseMessages = false): Promise<IMessage[] | BaseMessage[]> {
if (!this.redisClient) return []
const id = overrideSessionId ?? this.sessionId
const id = overrideSessionId ? overrideSessionId : this.sessionId
const rawStoredMessages = await this.redisClient.lrange(id, this.windowSize ? this.windowSize * -1 : 0, -1)
const orderedMessages = rawStoredMessages.reverse().map((message) => JSON.parse(message))
const baseMessages = orderedMessages.map(mapStoredMessageToChatMessage)
@@ -199,7 +199,7 @@ class BufferMemoryExtended extends FlowiseMemory implements MemoryMethods {
async addChatMessages(msgArray: { text: string; type: MessageType }[], overrideSessionId = ''): Promise<void> {
if (!this.redisClient) return
const id = overrideSessionId ?? this.sessionId
const id = overrideSessionId ? overrideSessionId : this.sessionId
const input = msgArray.find((msg) => msg.type === 'userMessage')
const output = msgArray.find((msg) => msg.type === 'apiMessage')
@@ -219,7 +219,7 @@ class BufferMemoryExtended extends FlowiseMemory implements MemoryMethods {
async clearChatMessages(overrideSessionId = ''): Promise<void> {
if (!this.redisClient) return
const id = overrideSessionId ?? this.sessionId
const id = overrideSessionId ? overrideSessionId : this.sessionId
await this.redisClient.del(id)
await this.clear()
}
packages/components/nodes/memory/UpstashRedisBackedChatMemory/UpstashRedisBackedChatMemory.ts
+3 -3
View File
@@ -114,7 +114,7 @@ class BufferMemoryExtended extends FlowiseMemory implements MemoryMethods {
async getChatMessages(overrideSessionId = '', returnBaseMessages = false): Promise<IMessage[] | BaseMessage[]> {
if (!this.redisClient) return []
const id = overrideSessionId ?? this.sessionId
const id = overrideSessionId ? overrideSessionId : this.sessionId
const rawStoredMessages: StoredMessage[] = await this.redisClient.lrange<StoredMessage>(id, 0, -1)
const orderedMessages = rawStoredMessages.reverse()
const previousMessages = orderedMessages.filter((x): x is StoredMessage => x.type !== undefined && x.data.content !== undefined)
@@ -125,7 +125,7 @@ class BufferMemoryExtended extends FlowiseMemory implements MemoryMethods {
async addChatMessages(msgArray: { text: string; type: MessageType }[], overrideSessionId = ''): Promise<void> {
if (!this.redisClient) return
const id = overrideSessionId ?? this.sessionId
const id = overrideSessionId ? overrideSessionId : this.sessionId
const input = msgArray.find((msg) => msg.type === 'userMessage')
const output = msgArray.find((msg) => msg.type === 'apiMessage')
@@ -145,7 +145,7 @@ class BufferMemoryExtended extends FlowiseMemory implements MemoryMethods {
async clearChatMessages(overrideSessionId = ''): Promise<void> {
if (!this.redisClient) return
const id = overrideSessionId ?? this.sessionId
const id = overrideSessionId ? overrideSessionId : this.sessionId
await this.redisClient.del(id)
await this.clear()
}
packages/components/nodes/memory/ZepMemory/ZepMemory.ts
+3 -3
View File
@@ -162,14 +162,14 @@ class ZepMemoryExtended extends ZepMemory implements MemoryMethods {
}
async getChatMessages(overrideSessionId = '', returnBaseMessages = false): Promise<IMessage[] | BaseMessage[]> {
const id = overrideSessionId ?? this.sessionId
const id = overrideSessionId ? overrideSessionId : this.sessionId
const memoryVariables = await this.loadMemoryVariables({}, id)
const baseMessages = memoryVariables[this.memoryKey]
return returnBaseMessages ? baseMessages : convertBaseMessagetoIMessage(baseMessages)
}
async addChatMessages(msgArray: { text: string; type: MessageType }[], overrideSessionId = ''): Promise<void> {
const id = overrideSessionId ?? this.sessionId
const id = overrideSessionId ? overrideSessionId : this.sessionId
const input = msgArray.find((msg) => msg.type === 'userMessage')
const output = msgArray.find((msg) => msg.type === 'apiMessage')
const inputValues = { [this.inputKey ?? 'input']: input?.text }
@@ -179,7 +179,7 @@ class ZepMemoryExtended extends ZepMemory implements MemoryMethods {
}
async clearChatMessages(overrideSessionId = ''): Promise<void> {
const id = overrideSessionId ?? this.sessionId
const id = overrideSessionId ? overrideSessionId : this.sessionId
await this.clear(id)
}
}
packages/components/nodes/outputparsers/StructuredOutputParserAdvanced/StructuredOutputParserAdvanced.ts
+79
View File
@@ -0,0 +1,79 @@
import { getBaseClasses, INode, INodeData, INodeParams } from '../../../src'
import { BaseOutputParser } from 'langchain/schema/output_parser'
import { StructuredOutputParser as LangchainStructuredOutputParser } from 'langchain/output_parsers'
import { CATEGORY } from '../OutputParserHelpers'
import { z } from 'zod'
class AdvancedStructuredOutputParser implements INode {
label: string
name: string
version: number
description: string
type: string
icon: string
category: string
baseClasses: string[]
inputs: INodeParams[]
credential: INodeParams
constructor() {
this.label = 'Advanced Structured Output Parser'
this.name = 'advancedStructuredOutputParser'
this.version = 1.0
this.type = 'AdvancedStructuredOutputParser'
this.description = 'Parse the output of an LLM call into a given structure by providing a Zod schema.'
this.icon = 'structure.svg'
this.category = CATEGORY
this.baseClasses = [this.type, ...getBaseClasses(BaseOutputParser)]
this.inputs = [
{
label: 'Autofix',
name: 'autofixParser',
type: 'boolean',
optional: true,
description: 'In the event that the first call fails, will make another call to the model to fix any errors.'
},
{
label: 'Example JSON',
name: 'exampleJson',
type: 'string',
description: 'Zod schema for the output of the model',
rows: 10,
default: `z.object({
title: z.string(), // Title of the movie as a string
yearOfRelease: z.number().int(), // Release year as an integer number,
genres: z.enum([
"Action", "Comedy", "Drama", "Fantasy", "Horror",
"Mystery", "Romance", "Science Fiction", "Thriller", "Documentary"
]).array().max(2), // Array of genres, max of 2 from the defined enum
shortDescription: z.string().max(500) // Short description, max 500 characters
})`
}
]
}
async init(nodeData: INodeData): Promise<any> {
const schemaString = nodeData.inputs?.exampleJson as string
const autoFix = nodeData.inputs?.autofixParser as boolean
const zodSchemaFunction = new Function('z', `return ${schemaString}`)
const zodSchema = zodSchemaFunction(z)
try {
const structuredOutputParser = LangchainStructuredOutputParser.fromZodSchema(zodSchema)
// NOTE: When we change Flowise to return a json response, the following has to be changed to: JsonStructuredOutputParser
Object.defineProperty(structuredOutputParser, 'autoFix', {
enumerable: true,
configurable: true,
writable: true,
value: autoFix
})
return structuredOutputParser
} catch (exception) {
throw new Error('Error parsing Zod Schema: ' + exception)
}
}
}
module.exports = { nodeClass: AdvancedStructuredOutputParser }
packages/components/nodes/outputparsers/StructuredOutputParserAdvanced/structure.svg
+8
View File
@@ -0,0 +1,8 @@
<svg width="32" height="32" viewBox="0 0 32 32" fill="none" xmlns="http://www.w3.org/2000/svg">
<path d="M16 3V13M16 3L13 6.13609M16 3L19 6.13609" stroke="black" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/>
<path d="M7 15V12C7 10.8954 7.89543 10 9 10H11M25 15V12C25 10.8954 24.1046 10 23 10H21" stroke="black" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/>
<path d="M12.5644 20.4399C11.6769 19.7608 9 19.6332 9 21.7961C9 24.1915 13 22.5657 13 25.0902C13 26.9875 10.33 27.5912 9 26.3537" stroke="black" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/>
<path d="M24 27V20L28 27V20" stroke="black" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/>
<path d="M16 23.5C16 20.7 17.6667 20 18.5 20C19.3333 20 21 20.7 21 23.5C21 26.3 19.3333 27 18.5 27C17.6667 27 16 26.3 16 23.5Z" stroke="black" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/>
<path d="M6 20V25C6 26.1046 5.10457 27 4 27V27C2.89543 27 2 26.1046 2 25V25" stroke="black" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 1.1 KiB

packages/components/nodes/retrievers/CohereRerankRetriever/CohereRerankRetriever.ts
+3 -1
View File
@@ -94,11 +94,13 @@ class CohereRerankRetriever_Retrievers implements INode {
{
label: 'Document',
name: 'document',
baseClasses: ['Document']
description: 'Array of document objects containing metadata and pageContent',
baseClasses: ['Document', 'json']
},
{
label: 'Text',
name: 'text',
description: 'Concatenated string from pageContent of documents',
baseClasses: ['string', 'json']
}
]
packages/components/nodes/retrievers/EmbeddingsFilterRetriever/EmbeddingsFilterRetriever.ts
+3 -1
View File
@@ -78,11 +78,13 @@ class EmbeddingsFilterRetriever_Retrievers implements INode {
{
label: 'Document',
name: 'document',
baseClasses: ['Document']
description: 'Array of document objects containing metadata and pageContent',
baseClasses: ['Document', 'json']
},
{
label: 'Text',
name: 'text',
description: 'Concatenated string from pageContent of documents',
baseClasses: ['string', 'json']
}
]
packages/components/nodes/retrievers/HydeRetriever/HydeRetriever.ts
+3 -1
View File
@@ -140,11 +140,13 @@ Passage:`
{
label: 'Document',
name: 'document',
baseClasses: ['Document']
description: 'Array of document objects containing metadata and pageContent',
baseClasses: ['Document', 'json']
},
{
label: 'Text',
name: 'text',
description: 'Concatenated string from pageContent of documents',
baseClasses: ['string', 'json']
}
]
packages/components/nodes/retrievers/LLMFilterRetriever/LLMFilterCompressionRetriever.ts
+3 -1
View File
@@ -58,11 +58,13 @@ class LLMFilterCompressionRetriever_Retrievers implements INode {
{
label: 'Document',
name: 'document',
baseClasses: ['Document']
description: 'Array of document objects containing metadata and pageContent',
baseClasses: ['Document', 'json']
},
{
label: 'Text',
name: 'text',
description: 'Concatenated string from pageContent of documents',
baseClasses: ['string', 'json']
}
]
packages/components/nodes/retrievers/RRFRetriever/RRFRetriever.ts
+3 -1
View File
@@ -89,11 +89,13 @@ class RRFRetriever_Retrievers implements INode {
{
label: 'Document',
name: 'document',
baseClasses: ['Document']
description: 'Array of document objects containing metadata and pageContent',
baseClasses: ['Document', 'json']
},
{
label: 'Text',
name: 'text',
description: 'Concatenated string from pageContent of documents',
baseClasses: ['string', 'json']
}
]
packages/components/nodes/retrievers/SimilarityThresholdRetriever/SimilarityThresholdRetriever.ts
+3 -1
View File
@@ -74,11 +74,13 @@ class SimilarityThresholdRetriever_Retrievers implements INode {
{
label: 'Document',
name: 'document',
baseClasses: ['Document']
description: 'Array of document objects containing metadata and pageContent',
baseClasses: ['Document', 'json']
},
{
label: 'Text',
name: 'text',
description: 'Concatenated string from pageContent of documents',
baseClasses: ['string', 'json']
}
]
packages/components/nodes/tools/CustomTool/core.ts
+1
View File
@@ -100,6 +100,7 @@ export class DynamicStructuredTool<
return result
}
// @ts-ignore
protected async _call(
arg: z.output<T>,
_?: CallbackManagerForToolRun,
packages/components/nodes/tools/QueryEngineTool/QueryEngineTool.ts
+8 -13
View File
@@ -1,5 +1,5 @@
import { INode, INodeData, INodeParams } from '../../../src/Interface'
import { VectorStoreIndex } from 'llamaindex'
import { BaseQueryEngine } from 'llamaindex'
class QueryEngine_Tools implements INode {
label: string
@@ -16,7 +16,7 @@ class QueryEngine_Tools implements INode {
constructor() {
this.label = 'QueryEngine Tool'
this.name = 'queryEngineToolLlamaIndex'
this.version = 1.0
this.version = 2.0
this.type = 'QueryEngineTool'
this.icon = 'queryEngineTool.svg'
this.category = 'Tools'
@@ -25,9 +25,9 @@ class QueryEngine_Tools implements INode {
this.baseClasses = [this.type]
this.inputs = [
{
label: 'Vector Store Index',
name: 'vectorStoreIndex',
type: 'VectorStoreIndex'
label: 'Base QueryEngine',
name: 'baseQueryEngine',
type: 'BaseQueryEngine'
},
{
label: 'Tool Name',
@@ -45,20 +45,15 @@ class QueryEngine_Tools implements INode {
}
async init(nodeData: INodeData): Promise<any> {
const vectorStoreIndex = nodeData.inputs?.vectorStoreIndex as VectorStoreIndex
const baseQueryEngine = nodeData.inputs?.baseQueryEngine as BaseQueryEngine
const toolName = nodeData.inputs?.toolName as string
const toolDesc = nodeData.inputs?.toolDesc as string
const queryEngineTool = {
queryEngine: vectorStoreIndex.asQueryEngine({
preFilters: {
...(vectorStoreIndex as any).metadatafilter
}
}),
queryEngine: baseQueryEngine,
metadata: {
name: toolName,
description: toolDesc
},
vectorStoreIndex
}
}
return queryEngineTool
packages/components/nodes/vectorstores/Postgres/Postgres.ts
+5 -20
View File
@@ -24,7 +24,7 @@ class Postgres_VectorStores implements INode {
constructor() {
this.label = 'Postgres'
this.name = 'postgres'
this.version = 2.0
this.version = 3.0
this.type = 'Postgres'
this.icon = 'postgres.svg'
this.category = 'Vector Stores'
@@ -60,13 +60,6 @@ class Postgres_VectorStores implements INode {
name: 'database',
type: 'string'
},
{
label: 'SSL Connection',
name: 'sslConnection',
type: 'boolean',
default: false,
optional: false
},
{
label: 'Port',
name: 'port',
@@ -124,7 +117,6 @@ class Postgres_VectorStores implements INode {
const docs = nodeData.inputs?.document as Document[]
const embeddings = nodeData.inputs?.embeddings as Embeddings
const additionalConfig = nodeData.inputs?.additionalConfig as string
const sslConnection = nodeData.inputs?.sslConnection as boolean
let additionalConfiguration = {}
if (additionalConfig) {
@@ -142,8 +134,7 @@ class Postgres_VectorStores implements INode {
port: nodeData.inputs?.port as number,
username: user,
password: password,
database: nodeData.inputs?.database as string,
ssl: sslConnection
database: nodeData.inputs?.database as string
}
const args = {
@@ -198,7 +189,8 @@ class Postgres_VectorStores implements INode {
type: 'postgres',
host: nodeData.inputs?.host as string,
port: nodeData.inputs?.port as number,
username: user,
username: user, // Required by TypeORMVectorStore
user: user, // Required by Pool in similaritySearchVectorWithScore
password: password,
database: nodeData.inputs?.database as string
}
@@ -248,14 +240,7 @@ const similaritySearchVectorWithScore = async (
ORDER BY "_distance" ASC
LIMIT $3;`
const poolOptions = {
host: postgresConnectionOptions.host,
port: postgresConnectionOptions.port,
user: postgresConnectionOptions.username,
password: postgresConnectionOptions.password,
database: postgresConnectionOptions.database
}
const pool = new Pool(poolOptions)
const pool = new Pool(postgresConnectionOptions)
const conn = await pool.connect()
const documents = await conn.query(queryString, [embeddingString, _filter, k])